From 3464df829e8b8a80a2a5b030f8b5ce8827688951 Mon Sep 17 00:00:00 2001 From: Alex Bee Date: Sun, 15 Dec 2024 12:35:16 +0100 Subject: [PATCH] ffmpeg: update v4l2-request patch Patch created using revisions b08d796..904a851 from branch v4l2-request-n7.1 of https://github.com/jernejsk/FFmpeg --- .../ffmpeg-001-v4l2-request.patch | 6206 ++++++++++------- 1 file changed, 3769 insertions(+), 2437 deletions(-) diff --git a/packages/multimedia/ffmpeg/patches/v4l2-request/ffmpeg-001-v4l2-request.patch b/packages/multimedia/ffmpeg/patches/v4l2-request/ffmpeg-001-v4l2-request.patch index fc75520688..663d4edcd4 100644 --- a/packages/multimedia/ffmpeg/patches/v4l2-request/ffmpeg-001-v4l2-request.patch +++ b/packages/multimedia/ffmpeg/patches/v4l2-request/ffmpeg-001-v4l2-request.patch @@ -1,103 +1,43 @@ -From a4530e188ec74bd227b0e2fe2bcd5f2160812d3d Mon Sep 17 00:00:00 2001 +From b32bc2026a32f1dea2688b619f1547781da6bdaa Mon Sep 17 00:00:00 2001 From: Jonas Karlman -Date: Mon, 3 Dec 2018 23:48:04 +0100 -Subject: [PATCH 01/11] avutil: add av_buffer_pool_flush() +Date: Tue, 6 Aug 2024 09:06:00 +0000 +Subject: [PATCH 01/11] avutil/hwcontext: Add hwdevice type for V4L2 Request + API -Used by V4L2 request API hwaccel +Add a hwdevice type for V4L2 Request API with transfer_data_from support +for AV_PIX_FMT_DRM_PRIME, based on AV_HWDEVICE_TYPE_DRM. + +AVV4L2RequestDeviceContext.media_fd can be set by the application or a +media device path can be supplied when hwdevice is created. When none +is supplied it default to -1 and hwaccel will auto-detect a media device +with a capable video device. Signed-off-by: Jonas Karlman --- - libavutil/buffer.c | 13 +++++++++++++ - libavutil/buffer.h | 5 +++++ - 2 files changed, 18 insertions(+) - -diff --git a/libavutil/buffer.c b/libavutil/buffer.c -index e4562a79b10e..09da632c009d 100644 ---- a/libavutil/buffer.c -+++ b/libavutil/buffer.c -@@ -319,6 +319,19 @@ static void buffer_pool_free(AVBufferPool *pool) - av_freep(&pool); - } - -+void av_buffer_pool_flush(AVBufferPool *pool) -+{ -+ ff_mutex_lock(&pool->mutex); -+ while (pool->pool) { -+ BufferPoolEntry *buf = pool->pool; -+ pool->pool = buf->next; -+ -+ buf->free(buf->opaque, buf->data); -+ av_freep(&buf); -+ } -+ ff_mutex_unlock(&pool->mutex); -+} -+ - void av_buffer_pool_uninit(AVBufferPool **ppool) - { - AVBufferPool *pool; -diff --git a/libavutil/buffer.h b/libavutil/buffer.h -index e1ef5b7f07fc..fde9bae4f60c 100644 ---- a/libavutil/buffer.h -+++ b/libavutil/buffer.h -@@ -284,6 +284,11 @@ AVBufferPool *av_buffer_pool_init2(size_t size, void *opaque, - AVBufferRef* (*alloc)(void *opaque, size_t size), - void (*pool_free)(void *opaque)); - -+/** -+ * Free all available buffers in a buffer pool. -+ */ -+ void av_buffer_pool_flush(AVBufferPool *pool); -+ - /** - * Mark the pool as being available for freeing. It will actually be freed only - * once all the allocated buffers associated with the pool are released. Thus it - -From 3c994397b3eeb2693dbf22fabe3808a227efb8a9 Mon Sep 17 00:00:00 2001 -From: Jonas Karlman -Date: Sat, 15 Dec 2018 22:32:16 +0100 -Subject: [PATCH 02/11] Add common V4L2 request API code - -Signed-off-by: Jonas Karlman -Signed-off-by: Alex Bee ---- - configure | 12 + - libavcodec/Makefile | 1 + - libavcodec/hwconfig.h | 2 + - libavcodec/v4l2_request.c | 1048 +++++++++++++++++++++++++++++++++++++ - libavcodec/v4l2_request.h | 78 +++ - 5 files changed, 1141 insertions(+) - create mode 100644 libavcodec/v4l2_request.c - create mode 100644 libavcodec/v4l2_request.h + configure | 7 + + libavutil/Makefile | 3 + + libavutil/hwcontext.c | 4 + + libavutil/hwcontext.h | 1 + + libavutil/hwcontext_internal.h | 1 + + libavutil/hwcontext_v4l2request.c | 261 ++++++++++++++++++++++++++++++ + libavutil/hwcontext_v4l2request.h | 41 +++++ + 7 files changed, 318 insertions(+) + create mode 100644 libavutil/hwcontext_v4l2request.c + create mode 100644 libavutil/hwcontext_v4l2request.h diff --git a/configure b/configure -index 3cd3bdfb4496..cbc5442c8a0e 100755 +index d77a55b653..b8f82f0fbf 100755 --- a/configure +++ b/configure -@@ -281,6 +281,7 @@ External library support: - if openssl, gnutls or mbedtls is not used [no] - --enable-libtwolame enable MP2 encoding via libtwolame [no] - --enable-libuavs3d enable AVS3 decoding via libuavs3d [no] -+ --enable-libudev enable libudev [no] - --enable-libv4l2 enable libv4l2/v4l-utils [no] - --enable-libvidstab enable video stabilization using vid.stab [no] - --enable-libvmaf enable vmaf filter via libvmaf [no] -@@ -350,6 +351,7 @@ External library support: +@@ -359,6 +359,7 @@ External library support: --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no] --enable-rkmpp enable Rockchip Media Process Platform code [no] --disable-v4l2-m2m disable V4L2 mem2mem code [autodetect] -+ --enable-v4l2-request enable V4L2 request API code [no] ++ --enable-v4l2-request enable V4L2 Request API code [no] --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect] --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect] --disable-videotoolbox disable VideoToolbox code [autodetect] -@@ -1857,6 +1859,7 @@ EXTERNAL_LIBRARY_LIST=" - libtheora - libtwolame - libuavs3d -+ libudev - libv4l2 - libvmaf - libvorbis -@@ -1913,6 +1916,7 @@ HWACCEL_LIBRARY_LIST=" +@@ -2025,6 +2026,7 @@ HWACCEL_LIBRARY_LIST=" mmal omx opencl @@ -105,73 +45,105 @@ index 3cd3bdfb4496..cbc5442c8a0e 100755 " DOCUMENT_LIST=" -@@ -2999,6 +3003,7 @@ d3d11va_deps="dxva_h ID3D11VideoDecoder ID3D11VideoContext" - dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" +@@ -3146,6 +3148,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" ffnvcodec_deps_any="libdl LoadLibrary" + mediacodec_deps="android mediandk" nvdec_deps="ffnvcodec" -+v4l2_request_deps="linux_videodev2_h linux_media_h v4l2_timeval_to_ns libdrm libudev" ++v4l2_request_deps="linux_media_h v4l2_timeval_to_ns" vaapi_x11_deps="xlib_x11" videotoolbox_hwaccel_deps="videotoolbox pthreads" videotoolbox_hwaccel_extralibs="-framework QuartzCore" -@@ -6692,6 +6697,7 @@ enabled libtwolame && require libtwolame twolame.h twolame_init -ltwolame - { check_lib libtwolame twolame.h twolame_encode_buffer_float32_interleaved -ltwolame || - die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; } - enabled libuavs3d && require_pkg_config libuavs3d "uavs3d >= 1.1.41" uavs3d.h uavs3d_decode -+enabled libudev && require_pkg_config libudev libudev libudev.h udev_new - enabled libv4l2 && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_ioctl - enabled libvidstab && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit - enabled libvmaf && require_pkg_config libvmaf "libvmaf >= 2.0.0" libvmaf.h vmaf_init -@@ -6794,6 +6800,10 @@ enabled rkmpp && { require_pkg_config rkmpp rockchip_mpp rockchip/r - { enabled libdrm || - die "ERROR: rkmpp requires --enable-libdrm"; } - } -+enabled v4l2_request && { enabled libdrm || -+ die "ERROR: v4l2-request requires --enable-libdrm"; } && -+ { enabled libudev || -+ die "ERROR: v4l2-request requires --enable-libudev"; } - enabled vapoursynth && require_pkg_config vapoursynth "vapoursynth-script >= 42" VSScript.h vsscript_init - - -@@ -6876,6 +6886,8 @@ if enabled v4l2_m2m; then +@@ -7197,6 +7200,10 @@ if enabled v4l2_m2m; then check_cc vp9_v4l2_m2m linux/videodev2.h "int i = V4L2_PIX_FMT_VP9;" fi -+check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns ++if enabled v4l2_request; then ++ check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns ++fi + check_headers sys/videoio.h test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete -diff --git a/libavcodec/Makefile b/libavcodec/Makefile -index 389253f5d08e..0148242ed0c3 100644 ---- a/libavcodec/Makefile -+++ b/libavcodec/Makefile -@@ -170,6 +170,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o - OBJS-$(CONFIG_VP56DSP) += vp56dsp.o - OBJS-$(CONFIG_VP8DSP) += vp8dsp.o - OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o -+OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o - OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o - OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o +diff --git a/libavutil/Makefile b/libavutil/Makefile +index 6e6fa8d800..1ce46157dd 100644 +--- a/libavutil/Makefile ++++ b/libavutil/Makefile +@@ -48,6 +48,7 @@ HEADERS = adler32.h \ + hwcontext_qsv.h \ + hwcontext_mediacodec.h \ + hwcontext_opencl.h \ ++ hwcontext_v4l2request.h \ + hwcontext_vaapi.h \ + hwcontext_videotoolbox.h \ + hwcontext_vdpau.h \ +@@ -201,6 +202,7 @@ OBJS-$(CONFIG_MACOS_KPERF) += macos_kperf.o + OBJS-$(CONFIG_MEDIACODEC) += hwcontext_mediacodec.o + OBJS-$(CONFIG_OPENCL) += hwcontext_opencl.o + OBJS-$(CONFIG_QSV) += hwcontext_qsv.o ++OBJS-$(CONFIG_V4L2_REQUEST) += hwcontext_v4l2request.o + OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o + OBJS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.o + OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o +@@ -222,6 +224,7 @@ SKIPHEADERS-$(CONFIG_D3D12VA) += hwcontext_d3d12va.h + SKIPHEADERS-$(CONFIG_DXVA2) += hwcontext_dxva2.h + SKIPHEADERS-$(CONFIG_QSV) += hwcontext_qsv.h + SKIPHEADERS-$(CONFIG_OPENCL) += hwcontext_opencl.h ++SKIPHEADERS-$(CONFIG_V4L2_REQUEST) += hwcontext_v4l2request.h + SKIPHEADERS-$(CONFIG_VAAPI) += hwcontext_vaapi.h + SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.h + SKIPHEADERS-$(CONFIG_VDPAU) += hwcontext_vdpau.h +diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c +index fa99a0d8a4..7fae9381da 100644 +--- a/libavutil/hwcontext.c ++++ b/libavutil/hwcontext.c +@@ -65,6 +65,9 @@ static const HWContextType * const hw_table[] = { + #endif + #if CONFIG_VULKAN + &ff_hwcontext_type_vulkan, ++#endif ++#if CONFIG_V4L2_REQUEST ++ &ff_hwcontext_type_v4l2request, + #endif + NULL, + }; +@@ -77,6 +80,7 @@ static const char *const hw_type_names[] = { + [AV_HWDEVICE_TYPE_D3D12VA] = "d3d12va", + [AV_HWDEVICE_TYPE_OPENCL] = "opencl", + [AV_HWDEVICE_TYPE_QSV] = "qsv", ++ [AV_HWDEVICE_TYPE_V4L2REQUEST] = "v4l2request", + [AV_HWDEVICE_TYPE_VAAPI] = "vaapi", + [AV_HWDEVICE_TYPE_VDPAU] = "vdpau", + [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox", +diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h +index bac30debae..8cf50ddbd0 100644 +--- a/libavutil/hwcontext.h ++++ b/libavutil/hwcontext.h +@@ -38,6 +38,7 @@ enum AVHWDeviceType { + AV_HWDEVICE_TYPE_MEDIACODEC, + AV_HWDEVICE_TYPE_VULKAN, + AV_HWDEVICE_TYPE_D3D12VA, ++ AV_HWDEVICE_TYPE_V4L2REQUEST, + }; -diff --git a/libavcodec/hwconfig.h b/libavcodec/hwconfig.h -index 721424912c46..00864efc27da 100644 ---- a/libavcodec/hwconfig.h -+++ b/libavcodec/hwconfig.h -@@ -78,6 +78,8 @@ typedef struct AVCodecHWConfigInternal { - HW_CONFIG_HWACCEL(1, 1, 1, VIDEOTOOLBOX, VIDEOTOOLBOX, ff_ ## codec ## _videotoolbox_hwaccel) - #define HWACCEL_D3D11VA(codec) \ - HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD, NONE, ff_ ## codec ## _d3d11va_hwaccel) -+#define HWACCEL_V4L2REQUEST(codec) \ -+ HW_CONFIG_HWACCEL(1, 0, 0, DRM_PRIME, DRM, ff_ ## codec ## _v4l2request_hwaccel) - - #define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \ - &(const AVCodecHWConfigInternal) { \ -diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c + /** +diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h +index e32b786238..fd0cf29c6e 100644 +--- a/libavutil/hwcontext_internal.h ++++ b/libavutil/hwcontext_internal.h +@@ -158,6 +158,7 @@ extern const HWContextType ff_hwcontext_type_drm; + extern const HWContextType ff_hwcontext_type_dxva2; + extern const HWContextType ff_hwcontext_type_opencl; + extern const HWContextType ff_hwcontext_type_qsv; ++extern const HWContextType ff_hwcontext_type_v4l2request; + extern const HWContextType ff_hwcontext_type_vaapi; + extern const HWContextType ff_hwcontext_type_vdpau; + extern const HWContextType ff_hwcontext_type_videotoolbox; +diff --git a/libavutil/hwcontext_v4l2request.c b/libavutil/hwcontext_v4l2request.c new file mode 100644 -index 000000000000..595a53e6d4d8 +index 0000000000..833fbf9f40 --- /dev/null -+++ b/libavcodec/v4l2_request.c -@@ -0,0 +1,1048 @@ ++++ b/libavutil/hwcontext_v4l2request.c +@@ -0,0 +1,261 @@ +/* + * This file is part of FFmpeg. + * @@ -190,20 +162,399 @@ index 000000000000..595a53e6d4d8 + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + -+#include ++#include "config.h" ++ +#include -+#include ++#include +#include +#include +#include -+#include -+#include -+#include +#include + ++#include "avassert.h" ++#include "hwcontext_drm.h" ++#include "hwcontext_internal.h" ++#include "hwcontext_v4l2request.h" ++#include "mem.h" ++ ++static void v4l2request_device_free(AVHWDeviceContext *hwdev) ++{ ++ AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx; ++ ++ if (hwctx->media_fd >= 0) { ++ close(hwctx->media_fd); ++ hwctx->media_fd = -1; ++ } ++} ++ ++static int v4l2request_device_create(AVHWDeviceContext *hwdev, const char *device, ++ AVDictionary *opts, int flags) ++{ ++ AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx; ++ ++ hwctx->media_fd = -1; ++ hwdev->free = v4l2request_device_free; ++ ++ // Use auto-detect ++ if (!device || !device[0]) ++ return 0; ++ ++ hwctx->media_fd = open(device, O_RDWR); ++ if (hwctx->media_fd < 0) ++ return AVERROR(errno); ++ ++ return 0; ++} ++ ++static int v4l2request_device_init(AVHWDeviceContext *hwdev) ++{ ++ AVV4L2RequestDeviceContext *hwctx = hwdev->hwctx; ++ struct media_device_info device_info; ++ ++ // Use auto-detect ++ if (hwctx->media_fd < 0) ++ return 0; ++ ++ if (ioctl(hwctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0) ++ return AVERROR(errno); ++ ++ av_log(hwdev, AV_LOG_VERBOSE, "Using V4L2 media driver %s (%u.%u.%u)\n", ++ device_info.driver, ++ device_info.driver_version >> 16, ++ (device_info.driver_version >> 8) & 0xff, ++ device_info.driver_version & 0xff); ++ ++ return 0; ++} ++ ++static int v4l2request_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) ++{ ++ frame->buf[0] = av_buffer_pool_get(hwfc->pool); ++ if (!frame->buf[0]) ++ return AVERROR(ENOMEM); ++ ++ frame->data[0] = (uint8_t *)frame->buf[0]->data; ++ ++ frame->format = AV_PIX_FMT_DRM_PRIME; ++ frame->width = hwfc->width; ++ frame->height = hwfc->height; ++ ++ return 0; ++} ++ ++typedef struct DRMMapping { ++ // Address and length of each mmap()ed region. ++ int nb_regions; ++ int object[AV_DRM_MAX_PLANES]; ++ void *address[AV_DRM_MAX_PLANES]; ++ size_t length[AV_DRM_MAX_PLANES]; ++} DRMMapping; ++ ++static void v4l2request_unmap_frame(AVHWFramesContext *hwfc, ++ HWMapDescriptor *hwmap) ++{ ++ DRMMapping *map = hwmap->priv; ++ ++ for (int i = 0; i < map->nb_regions; i++) { ++ struct dma_buf_sync sync = { ++ .flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ, ++ }; ++ ioctl(map->object[i], DMA_BUF_IOCTL_SYNC, &sync); ++ munmap(map->address[i], map->length[i]); ++ } ++ ++ av_free(map); ++} ++ ++static int v4l2request_map_frame(AVHWFramesContext *hwfc, ++ AVFrame *dst, const AVFrame *src) ++{ ++ const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0]; ++ struct dma_buf_sync sync = { ++ .flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ, ++ }; ++ DRMMapping *map; ++ int ret, i, p, plane; ++ void *addr; ++ ++ map = av_mallocz(sizeof(*map)); ++ if (!map) ++ return AVERROR(ENOMEM); ++ ++ av_assert0(desc->nb_objects <= AV_DRM_MAX_PLANES); ++ for (i = 0; i < desc->nb_objects; i++) { ++ addr = mmap(NULL, desc->objects[i].size, AV_HWFRAME_MAP_READ, MAP_SHARED, ++ desc->objects[i].fd, 0); ++ if (addr == MAP_FAILED) { ++ av_log(hwfc, AV_LOG_ERROR, "Failed to map DRM object %d to memory: %s (%d)\n", ++ desc->objects[i].fd, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ map->address[i] = addr; ++ map->length[i] = desc->objects[i].size; ++ map->object[i] = desc->objects[i].fd; ++ ++ /* ++ * We're not checking for errors here because the kernel may not ++ * support the ioctl, in which case its okay to carry on ++ */ ++ ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_SYNC, &sync); ++ } ++ map->nb_regions = i; ++ ++ plane = 0; ++ for (i = 0; i < desc->nb_layers; i++) { ++ const AVDRMLayerDescriptor *layer = &desc->layers[i]; ++ for (p = 0; p < layer->nb_planes; p++) { ++ dst->data[plane] = ++ (uint8_t *)map->address[layer->planes[p].object_index] + ++ layer->planes[p].offset; ++ dst->linesize[plane] = layer->planes[p].pitch; ++ ++plane; ++ } ++ } ++ av_assert0(plane <= AV_DRM_MAX_PLANES); ++ ++ dst->width = src->width; ++ dst->height = src->height; ++ ++ ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, ++ v4l2request_unmap_frame, map); ++ if (ret < 0) ++ goto fail; ++ ++ return 0; ++ ++fail: ++ for (i = 0; i < desc->nb_objects; i++) { ++ if (map->address[i]) ++ munmap(map->address[i], map->length[i]); ++ } ++ av_free(map); ++ return ret; ++} ++ ++static int v4l2request_transfer_get_formats(AVHWFramesContext *hwfc, ++ enum AVHWFrameTransferDirection dir, ++ enum AVPixelFormat **formats) ++{ ++ enum AVPixelFormat *pix_fmts; ++ ++ if (dir == AV_HWFRAME_TRANSFER_DIRECTION_TO) ++ return AVERROR(ENOSYS); ++ ++ pix_fmts = av_malloc_array(2, sizeof(*pix_fmts)); ++ if (!pix_fmts) ++ return AVERROR(ENOMEM); ++ ++ pix_fmts[0] = hwfc->sw_format; ++ pix_fmts[1] = AV_PIX_FMT_NONE; ++ ++ *formats = pix_fmts; ++ return 0; ++} ++ ++static int v4l2request_transfer_data_from(AVHWFramesContext *hwfc, ++ AVFrame *dst, const AVFrame *src) ++{ ++ AVFrame *map; ++ int ret; ++ ++ if (dst->width > hwfc->width || dst->height > hwfc->height) ++ return AVERROR(EINVAL); ++ ++ map = av_frame_alloc(); ++ if (!map) ++ return AVERROR(ENOMEM); ++ map->format = dst->format; ++ ++ ret = v4l2request_map_frame(hwfc, map, src); ++ if (ret) ++ goto fail; ++ ++ map->width = dst->width; ++ map->height = dst->height; ++ ++ ret = av_frame_copy(dst, map); ++ if (ret) ++ goto fail; ++ ++ ret = 0; ++fail: ++ av_frame_free(&map); ++ return ret; ++} ++ ++const HWContextType ff_hwcontext_type_v4l2request = { ++ .type = AV_HWDEVICE_TYPE_V4L2REQUEST, ++ .name = "V4L2 Request API", ++ ++ .device_hwctx_size = sizeof(AVV4L2RequestDeviceContext), ++ .device_create = v4l2request_device_create, ++ .device_init = v4l2request_device_init, ++ ++ .frames_get_buffer = v4l2request_get_buffer, ++ ++ .transfer_get_formats = v4l2request_transfer_get_formats, ++ .transfer_data_from = v4l2request_transfer_data_from, ++ ++ .pix_fmts = (const enum AVPixelFormat[]) { ++ AV_PIX_FMT_DRM_PRIME, ++ AV_PIX_FMT_NONE ++ }, ++}; +diff --git a/libavutil/hwcontext_v4l2request.h b/libavutil/hwcontext_v4l2request.h +new file mode 100644 +index 0000000000..0fe42f97b4 +--- /dev/null ++++ b/libavutil/hwcontext_v4l2request.h +@@ -0,0 +1,41 @@ ++/* ++ * This file is part of FFmpeg. ++ * ++ * FFmpeg is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2.1 of the License, or (at your option) any later version. ++ * ++ * FFmpeg is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with FFmpeg; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#ifndef AVUTIL_HWCONTEXT_V4L2REQUEST_H ++#define AVUTIL_HWCONTEXT_V4L2REQUEST_H ++ ++/** ++ * @file ++ * An API-specific header for AV_HWDEVICE_TYPE_V4L2REQUEST. ++ */ ++ ++/** ++ * V4L2 Request API device details. ++ * ++ * Allocated as AVHWDeviceContext.hwctx ++ */ ++typedef struct AVV4L2RequestDeviceContext { ++ /** ++ * File descriptor of media device. ++ * ++ * Defaults to -1 for auto-detect. ++ */ ++ int media_fd; ++} AVV4L2RequestDeviceContext; ++ ++#endif /* AVUTIL_HWCONTEXT_V4L2REQUEST_H */ + +From 8b12b1ca49f86dee00a68d819990a2d63150ac1a Mon Sep 17 00:00:00 2001 +From: Jonas Karlman +Date: Tue, 6 Aug 2024 09:06:01 +0000 +Subject: [PATCH 02/11] avcodec: Add common V4L2 Request API code + +Add common helpers for supporting V4L2 Request API hwaccels. + +Basic flow for initialization follow the kernel Memory-to-memory +Stateless Video Decoder Interface > Initialization [1]. + +In init video devices is probed and when a capable device is found it is +initialized for decoding. Codec specific CONTROLs may be set to assist +kernel driver. A supported CAPTURE format is selected. A hw frame ctx is +created and CAPTURE buffers is allocated. OUTPUT buffers and request +objects for a circular queue is also allocated. + +In frame_params a buffer pool is created and configured to allocate +CAPTURE buffers. + +In flush all queued OUTPUT and CAPTURE buffers is dequeued. + +In uninit any pending request is flushed, also video and media device is +closed. The media device is not closed when ownership of the media_fd +has been transfered to the hwdevice. + +[1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#initialization + +Co-developed-by: Jernej Skrabec +Signed-off-by: Jernej Skrabec +Co-developed-by: Alex Bee +Signed-off-by: Alex Bee +Signed-off-by: Jonas Karlman +--- + MAINTAINERS | 1 + + libavcodec/Makefile | 1 + + libavcodec/v4l2_request.c | 441 +++++++++++++++++++++++++++++ + libavcodec/v4l2_request.h | 84 ++++++ + libavcodec/v4l2_request_internal.h | 42 +++ + 5 files changed, 569 insertions(+) + create mode 100644 libavcodec/v4l2_request.c + create mode 100644 libavcodec/v4l2_request.h + create mode 100644 libavcodec/v4l2_request_internal.h + +diff --git a/MAINTAINERS b/MAINTAINERS +index 76651d5ff8..391acfbe16 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -278,6 +278,7 @@ Hardware acceleration: + d3d11va* Steve Lhomme + d3d12va_encode* Tong Wu + mediacodec* Matthieu Bouron, Aman Gupta, Zhao Zhili ++ v4l2_request* Jonas Karlman (CC jonas@kwiboo.se) + vaapi* Haihao Xiang + vaapi_encode* Mark Thompson, Haihao Xiang + vdpau* Philip Langdale, Carl Eugen Hoyos +diff --git a/libavcodec/Makefile b/libavcodec/Makefile +index a4fcce3b42..af60923827 100644 +--- a/libavcodec/Makefile ++++ b/libavcodec/Makefile +@@ -176,6 +176,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o + OBJS-$(CONFIG_VP56DSP) += vp56dsp.o + OBJS-$(CONFIG_VP8DSP) += vp8dsp.o + OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o ++OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o + OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o + OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o + +diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c +new file mode 100644 +index 0000000000..436c8de4a4 +--- /dev/null ++++ b/libavcodec/v4l2_request.c +@@ -0,0 +1,441 @@ ++/* ++ * This file is part of FFmpeg. ++ * ++ * FFmpeg is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2.1 of the License, or (at your option) any later version. ++ * ++ * FFmpeg is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with FFmpeg; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#include "config.h" ++ ++#include ++#include ++#include ++#include ++ ++#include "libavutil/hwcontext_v4l2request.h" ++#include "libavutil/mem.h" +#include "decode.h" -+#include "internal.h" -+#include "v4l2_request.h" ++#include "v4l2_request_internal.h" + +static const AVClass v4l2_request_context_class = { + .class_name = "V4L2RequestContext", @@ -211,33 +562,40 @@ index 000000000000..595a53e6d4d8 + .version = LIBAVUTIL_VERSION_INT, +}; + -+uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame) ++int ff_v4l2_request_query_control(AVCodecContext *avctx, ++ struct v4l2_query_ext_ctrl *control) +{ -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)frame->data[0]; -+ return req ? v4l2_timeval_to_ns(&req->capture.buffer.timestamp) : 0; -+} ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); + -+int ff_v4l2_request_reset_frame(AVCodecContext *avctx, AVFrame *frame) -+{ -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)frame->data[0]; -+ memset(&req->drm, 0, sizeof(AVDRMFrameDescriptor)); -+ req->output.used = 0; -+ return 0; -+} -+ -+int ff_v4l2_request_append_output_buffer(AVCodecContext *avctx, AVFrame *frame, const uint8_t *data, uint32_t size) -+{ -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)frame->data[0]; -+ if (req->output.used + size + (AV_INPUT_BUFFER_PADDING_SIZE * 4) <= req->output.size) { -+ memcpy(req->output.addr + req->output.used, data, size); -+ req->output.used += size; -+ } else { -+ av_log(avctx, AV_LOG_ERROR, "%s: output.used=%u output.size=%u size=%u\n", __func__, req->output.used, req->output.size, size); ++ if (ioctl(ctx->video_fd, VIDIOC_QUERY_EXT_CTRL, control) < 0) { ++ // Skip error logging when driver does not support control id (EINVAL) ++ if (errno != EINVAL) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to query control %u: %s (%d)\n", ++ control->id, strerror(errno), errno); ++ } ++ return AVERROR(errno); + } ++ + return 0; +} + -+static int v4l2_request_controls(V4L2RequestContext *ctx, int request_fd, unsigned long type, struct v4l2_ext_control *control, int count) ++int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx, ++ uint32_t id) ++{ ++ struct v4l2_query_ext_ctrl control = { ++ .id = id, ++ }; ++ int ret; ++ ++ ret = ff_v4l2_request_query_control(avctx, &control); ++ if (ret < 0) ++ return ret; ++ ++ return control.default_value; ++} ++ ++int ff_v4l2_request_set_request_controls(V4L2RequestContext *ctx, int request_fd, ++ struct v4l2_ext_control *control, int count) +{ + struct v4l2_ext_controls controls = { + .controls = control, @@ -249,751 +607,31 @@ index 000000000000..595a53e6d4d8 + if (!control || !count) + return 0; + -+ return ioctl(ctx->video_fd, type, &controls); ++ if (ioctl(ctx->video_fd, VIDIOC_S_EXT_CTRLS, &controls) < 0) ++ return AVERROR(errno); ++ ++ return 0; +} + -+static int v4l2_request_set_controls(V4L2RequestContext *ctx, int request_fd, struct v4l2_ext_control *control, int count) ++int ff_v4l2_request_set_controls(AVCodecContext *avctx, ++ struct v4l2_ext_control *control, int count) +{ -+ return v4l2_request_controls(ctx, request_fd, VIDIOC_S_EXT_CTRLS, control, count); -+} -+ -+int ff_v4l2_request_set_controls(AVCodecContext *avctx, struct v4l2_ext_control *control, int count) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); + int ret; + -+ ret = v4l2_request_controls(ctx, -1, VIDIOC_S_EXT_CTRLS, control, count); ++ ret = ff_v4l2_request_set_request_controls(ctx, -1, control, count); + if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: set controls failed, %s (%d)\n", __func__, strerror(errno), errno); -+ return AVERROR(EINVAL); ++ av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s): %s (%d)\n", ++ count, strerror(errno), errno); + } + + return ret; +} + -+int ff_v4l2_request_get_controls(AVCodecContext *avctx, struct v4l2_ext_control *control, int count) ++static int v4l2_request_buffer_alloc(V4L2RequestContext *ctx, ++ V4L2RequestBuffer *buf, ++ enum v4l2_buf_type type) +{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ int ret; -+ -+ ret = v4l2_request_controls(ctx, -1, VIDIOC_G_EXT_CTRLS, control, count); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get controls failed, %s (%d)\n", __func__, strerror(errno), errno); -+ return AVERROR(EINVAL); -+ } -+ -+ return ret; -+} -+ -+int ff_v4l2_request_query_control(AVCodecContext *avctx, struct v4l2_query_ext_ctrl *control) -+{ -+ int ret; -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_QUERY_EXT_CTRL, control); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: query control failed, %s (%d)\n", __func__, strerror(errno), errno); -+ return AVERROR(EINVAL); -+ } -+ -+ return 0; -+} -+ -+int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx, uint32_t id) -+{ -+ int ret; -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ struct v4l2_queryctrl control = { -+ .id = id, -+ }; -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_QUERYCTRL, &control); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: query control failed, %s (%d)\n", __func__, strerror(errno), errno); -+ return AVERROR(EINVAL); -+ } -+ -+ return control.default_value; -+} -+ -+static int v4l2_request_queue_buffer(V4L2RequestContext *ctx, int request_fd, V4L2RequestBuffer *buf, uint32_t flags) -+{ -+ struct v4l2_plane planes[1] = {}; -+ struct v4l2_buffer buffer = { -+ .type = buf->buffer.type, -+ .memory = buf->buffer.memory, -+ .index = buf->index, -+ .timestamp.tv_usec = buf->index + 1, -+ .bytesused = buf->used, -+ .request_fd = request_fd, -+ .flags = ((request_fd >= 0) ? V4L2_BUF_FLAG_REQUEST_FD : 0) | flags, -+ }; -+ -+ if (V4L2_TYPE_IS_MULTIPLANAR(buf->buffer.type)) { -+ planes[0].bytesused = buf->used; -+ buffer.bytesused = 0; -+ buffer.length = 1; -+ buffer.m.planes = planes; -+ } -+ -+ return ioctl(ctx->video_fd, VIDIOC_QBUF, &buffer); -+} -+ -+static int v4l2_request_dequeue_buffer(V4L2RequestContext *ctx, V4L2RequestBuffer *buf) -+{ -+ int ret; -+ struct v4l2_plane planes[1] = {}; -+ struct v4l2_buffer buffer = { -+ .type = buf->buffer.type, -+ .memory = buf->buffer.memory, -+ .index = buf->index, -+ }; -+ -+ if (V4L2_TYPE_IS_MULTIPLANAR(buf->buffer.type)) { -+ buffer.length = 1; -+ buffer.m.planes = planes; -+ } -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_DQBUF, &buffer); -+ if (ret < 0) -+ return ret; -+ -+ buf->buffer.timestamp = buffer.timestamp; -+ return 0; -+} -+ -+static const struct { -+ uint32_t pixelformat; -+ enum AVPixelFormat sw_format; -+ uint32_t drm_format; -+ uint64_t format_modifier; -+} v4l2_request_capture_pixelformats[] = { -+ { V4L2_PIX_FMT_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_LINEAR }, -+ { V4L2_PIX_FMT_SUNXI_TILED_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_ALLWINNER_TILED }, -+}; -+ -+static int v4l2_request_set_drm_descriptor(V4L2RequestDescriptor *req, struct v4l2_format *format) -+{ -+ AVDRMFrameDescriptor *desc = &req->drm; -+ AVDRMLayerDescriptor *layer = &desc->layers[0]; -+ uint32_t pixelformat = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? format->fmt.pix_mp.pixelformat : format->fmt.pix.pixelformat; -+ -+ layer->format = 0; -+ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { -+ if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) { -+ layer->format = v4l2_request_capture_pixelformats[i].drm_format; -+ desc->objects[0].format_modifier = v4l2_request_capture_pixelformats[i].format_modifier; -+ break; -+ } -+ } -+ -+ if (!layer->format) -+ return -1; -+ -+ desc->nb_objects = 1; -+ desc->objects[0].fd = req->capture.fd; -+ desc->objects[0].size = req->capture.size; -+ -+ desc->nb_layers = 1; -+ layer->nb_planes = 2; -+ -+ layer->planes[0].object_index = 0; -+ layer->planes[0].offset = 0; -+ layer->planes[0].pitch = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? format->fmt.pix_mp.plane_fmt[0].bytesperline : format->fmt.pix.bytesperline; -+ -+ layer->planes[1].object_index = 0; -+ layer->planes[1].offset = layer->planes[0].pitch * (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? format->fmt.pix_mp.height : format->fmt.pix.height); -+ layer->planes[1].pitch = layer->planes[0].pitch; -+ -+ return 0; -+} -+ -+static int v4l2_request_queue_decode(AVCodecContext *avctx, AVFrame *frame, struct v4l2_ext_control *control, int count, int first_slice, int last_slice) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)frame->data[0]; -+ struct timeval tv = { 2, 0 }; -+ fd_set except_fds; -+ int ret; -+ -+ av_log(avctx, AV_LOG_DEBUG, "%s: ctx=%p used=%u controls=%d index=%d fd=%d request_fd=%d first_slice=%d last_slice=%d\n", __func__, -+ ctx, req->output.used, count, req->capture.index, req->capture.fd, req->request_fd, first_slice, last_slice); -+ -+ ret = v4l2_request_set_controls(ctx, req->request_fd, control, count); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: set controls failed for request %d, %s (%d)\n", __func__, req->request_fd, strerror(errno), errno); -+ return -1; -+ } -+ -+ memset(req->output.addr + req->output.used, 0, AV_INPUT_BUFFER_PADDING_SIZE * 4); -+ -+ ret = v4l2_request_queue_buffer(ctx, req->request_fd, &req->output, last_slice ? 0 : V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: queue output buffer %d failed for request %d, %s (%d)\n", __func__, req->output.index, req->request_fd, strerror(errno), errno); -+ return -1; -+ } -+ -+ if (first_slice) { -+ ret = v4l2_request_queue_buffer(ctx, -1, &req->capture, 0); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: queue capture buffer %d failed for request %d, %s (%d)\n", __func__, req->capture.index, req->request_fd, strerror(errno), errno); -+ return -1; -+ } -+ } -+ -+ // NOTE: do we need to dequeue when request fails/timeout? -+ -+ // 4. queue request and wait -+ ret = ioctl(req->request_fd, MEDIA_REQUEST_IOC_QUEUE, NULL); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: queue request %d failed, %s (%d)\n", __func__, req->request_fd, strerror(errno), errno); -+ goto fail; -+ } -+ -+ FD_ZERO(&except_fds); -+ FD_SET(req->request_fd, &except_fds); -+ -+ ret = select(req->request_fd + 1, NULL, NULL, &except_fds, &tv); -+ if (ret == 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: request %d timeout\n", __func__, req->request_fd); -+ goto fail; -+ } else if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: select request %d failed, %s (%d)\n", __func__, req->request_fd, strerror(errno), errno); -+ goto fail; -+ } -+ -+ ret = v4l2_request_dequeue_buffer(ctx, &req->output); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: dequeue output buffer %d failed for request %d, %s (%d)\n", __func__, req->output.index, req->request_fd, strerror(errno), errno); -+ return -1; -+ } -+ -+ if (last_slice) { -+ ret = v4l2_request_dequeue_buffer(ctx, &req->capture); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: dequeue capture buffer %d failed for request %d, %s (%d)\n", __func__, req->capture.index, req->request_fd, strerror(errno), errno); -+ return -1; -+ } -+ } -+ -+ // TODO: check errors -+ // buffer.flags & V4L2_BUF_FLAG_ERROR -+ -+ ret = ioctl(req->request_fd, MEDIA_REQUEST_IOC_REINIT, NULL); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: reinit request %d failed, %s (%d)\n", __func__, req->request_fd, strerror(errno), errno); -+ return -1; -+ } -+ -+ if (last_slice) -+ return v4l2_request_set_drm_descriptor(req, &ctx->format); -+ -+ return 0; -+ -+fail: -+ ret = v4l2_request_dequeue_buffer(ctx, &req->output); -+ if (ret < 0) -+ av_log(avctx, AV_LOG_ERROR, "%s: dequeue output buffer %d failed for request %d, %s (%d)\n", __func__, req->output.index, req->request_fd, strerror(errno), errno); -+ -+ ret = v4l2_request_dequeue_buffer(ctx, &req->capture); -+ if (ret < 0) -+ av_log(avctx, AV_LOG_ERROR, "%s: dequeue capture buffer %d failed for request %d, %s (%d)\n", __func__, req->capture.index, req->request_fd, strerror(errno), errno); -+ -+ ret = ioctl(req->request_fd, MEDIA_REQUEST_IOC_REINIT, NULL); -+ if (ret < 0) -+ av_log(avctx, AV_LOG_ERROR, "%s: reinit request %d failed, %s (%d)\n", __func__, req->request_fd, strerror(errno), errno); -+ -+ return -1; -+} -+ -+int ff_v4l2_request_decode_slice(AVCodecContext *avctx, AVFrame *frame, struct v4l2_ext_control *control, int count, int first_slice, int last_slice) -+{ -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)frame->data[0]; -+ -+ // fall back to queue each slice as a full frame -+ if ((req->output.capabilities & V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) != V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) -+ return v4l2_request_queue_decode(avctx, frame, control, count, 1, 1); -+ -+ return v4l2_request_queue_decode(avctx, frame, control, count, first_slice, last_slice); -+} -+ -+int ff_v4l2_request_decode_frame(AVCodecContext *avctx, AVFrame *frame, struct v4l2_ext_control *control, int count) -+{ -+ return v4l2_request_queue_decode(avctx, frame, control, count, 1, 1); -+} -+ -+static int v4l2_request_try_framesize(AVCodecContext *avctx, uint32_t pixelformat) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ struct v4l2_frmsizeenum frmsize = { -+ .index = 0, -+ .pixel_format = pixelformat, -+ }; -+ -+ if (ioctl(ctx->video_fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) < 0) -+ return 0; -+ -+ /* -+ * We only validate min/max framesize for V4L2_FRMSIZE_TYPE_STEPWISE here, since the alignment -+ * which is eventually needed will be done driver-side later in VIDIOC_S_FMT and there is no need -+ * validate step_width/step_height here -+ */ -+ do { -+ if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE && frmsize.discrete.width == avctx->coded_width && -+ frmsize.discrete.height == avctx->coded_height) -+ return 0; -+ else if ((frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE || frmsize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) && -+ avctx->coded_width >= frmsize.stepwise.min_width && avctx->coded_height >= frmsize.stepwise.min_height && -+ avctx->coded_width <= frmsize.stepwise.max_width && avctx->coded_height <= frmsize.stepwise.max_height) -+ return 0; -+ -+ frmsize.index++; -+ } while (ioctl(ctx->video_fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) >= 0); -+ -+ av_log(avctx, AV_LOG_INFO, "%s: pixelformat %s not supported for width %u height %u\n", __func__, av_fourcc2str(pixelformat), avctx->coded_width, avctx->coded_height); -+ return -1; -+} -+ -+static int v4l2_request_try_format(AVCodecContext *avctx, enum v4l2_buf_type type, uint32_t pixelformat) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ struct v4l2_fmtdesc fmtdesc = { -+ .index = 0, -+ .type = type, -+ }; -+ -+ if (V4L2_TYPE_IS_OUTPUT(type)) { -+ struct v4l2_create_buffers buffers = { -+ .count = 0, -+ .memory = V4L2_MEMORY_MMAP, -+ .format.type = type, -+ }; -+ -+ if (ioctl(ctx->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: create buffers failed for type %u, %s (%d)\n", __func__, type, strerror(errno), errno); -+ return -1; -+ } -+ -+ if ((buffers.capabilities & V4L2_BUF_CAP_SUPPORTS_REQUESTS) != V4L2_BUF_CAP_SUPPORTS_REQUESTS) { -+ av_log(avctx, AV_LOG_INFO, "%s: output buffer type do not support requests, capabilities %u\n", __func__, buffers.capabilities); -+ return -1; -+ } -+ } -+ -+ while (ioctl(ctx->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) { -+ if (fmtdesc.pixelformat == pixelformat) -+ return 0; -+ -+ fmtdesc.index++; -+ } -+ -+ av_log(avctx, AV_LOG_DEBUG, "%s: pixelformat %s not supported for type %u\n", __func__, av_fourcc2str(pixelformat), type); -+ return -1; -+} -+ -+static int v4l2_request_set_format(AVCodecContext *avctx, enum v4l2_buf_type type, uint32_t pixelformat, uint32_t buffersize) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ struct v4l2_format format = { -+ .type = type, -+ }; -+ -+ if (V4L2_TYPE_IS_MULTIPLANAR(type)) { -+ format.fmt.pix_mp.width = avctx->coded_width; -+ format.fmt.pix_mp.height = avctx->coded_height; -+ format.fmt.pix_mp.pixelformat = pixelformat; -+ format.fmt.pix_mp.plane_fmt[0].sizeimage = buffersize; -+ format.fmt.pix_mp.num_planes = 1; -+ } else { -+ format.fmt.pix.width = avctx->coded_width; -+ format.fmt.pix.height = avctx->coded_height; -+ format.fmt.pix.pixelformat = pixelformat; -+ format.fmt.pix.sizeimage = buffersize; -+ } -+ -+ return ioctl(ctx->video_fd, VIDIOC_S_FMT, &format); -+} -+ -+static int v4l2_request_select_capture_format(AVCodecContext *avctx) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ enum v4l2_buf_type type = ctx->format.type; -+ -+#if 1 -+ struct v4l2_format format = { -+ .type = type, -+ }; -+ struct v4l2_fmtdesc fmtdesc = { -+ .index = 0, -+ .type = type, -+ }; -+ uint32_t pixelformat; -+ -+ if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &format) < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get capture format failed, %s (%d)\n", __func__, strerror(errno), errno); -+ return -1; -+ } -+ -+ pixelformat = V4L2_TYPE_IS_MULTIPLANAR(type) ? format.fmt.pix_mp.pixelformat : format.fmt.pix.pixelformat; -+ -+ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { -+ if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) -+ return v4l2_request_set_format(avctx, type, pixelformat, 0); -+ } -+ -+ while (ioctl(ctx->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) { -+ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { -+ if (fmtdesc.pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) -+ return v4l2_request_set_format(avctx, type, fmtdesc.pixelformat, 0); -+ } -+ -+ fmtdesc.index++; -+ } -+#else -+ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { -+ uint32_t pixelformat = v4l2_request_capture_pixelformats[i].pixelformat; -+ if (!v4l2_request_try_format(avctx, type, pixelformat)) -+ return v4l2_request_set_format(avctx, type, pixelformat, 0); -+ } -+#endif -+ -+ return -1; -+} -+ -+static int v4l2_request_probe_video_device(struct udev_device *device, AVCodecContext *avctx, uint32_t pixelformat, uint32_t buffersize, struct v4l2_ext_control *control, int count) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ int ret = AVERROR(EINVAL); -+ struct v4l2_capability capability = {0}; -+ unsigned int capabilities = 0; -+ -+ const char *path = udev_device_get_devnode(device); -+ if (!path) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get video device devnode failed\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ctx->video_fd = open(path, O_RDWR | O_NONBLOCK, 0); -+ if (ctx->video_fd < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: opening %s failed, %s (%d)\n", __func__, path, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_QUERYCAP, &capability); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get video capability failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ if (capability.capabilities & V4L2_CAP_DEVICE_CAPS) -+ capabilities = capability.device_caps; -+ else -+ capabilities = capability.capabilities; -+ -+ av_log(avctx, AV_LOG_DEBUG, "%s: ctx=%p path=%s capabilities=%u\n", __func__, ctx, path, capabilities); -+ -+ if ((capabilities & V4L2_CAP_STREAMING) != V4L2_CAP_STREAMING) { -+ av_log(avctx, AV_LOG_ERROR, "%s: missing required streaming capability\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ if ((capabilities & V4L2_CAP_VIDEO_M2M_MPLANE) == V4L2_CAP_VIDEO_M2M_MPLANE) { -+ ctx->output_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; -+ ctx->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; -+ } else if ((capabilities & V4L2_CAP_VIDEO_M2M) == V4L2_CAP_VIDEO_M2M) { -+ ctx->output_type = V4L2_BUF_TYPE_VIDEO_OUTPUT; -+ ctx->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; -+ } else { -+ av_log(avctx, AV_LOG_ERROR, "%s: missing required mem2mem capability\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = v4l2_request_try_format(avctx, ctx->output_type, pixelformat); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_DEBUG, "%s: try output format failed\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = v4l2_request_try_framesize(avctx, pixelformat); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_DEBUG, "%s: try framesize failed\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = v4l2_request_set_format(avctx, ctx->output_type, pixelformat, buffersize); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: set output format failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = v4l2_request_set_controls(ctx, -1, control, count); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_DEBUG, "%s: set controls failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = v4l2_request_select_capture_format(avctx); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_DEBUG, "%s: select capture format failed\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ return 0; -+ -+fail: -+ if (ctx->video_fd >= 0) { -+ close(ctx->video_fd); -+ ctx->video_fd = -1; -+ } -+ return ret; -+} -+ -+static int v4l2_request_init_context(AVCodecContext *avctx) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ int ret; -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_G_FMT, &ctx->format); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get capture format failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ if (V4L2_TYPE_IS_MULTIPLANAR(ctx->format.type)) { -+ struct v4l2_pix_format_mplane *fmt = &ctx->format.fmt.pix_mp; -+ av_log(ctx, AV_LOG_DEBUG, "%s: pixelformat=%s width=%u height=%u bytesperline=%u sizeimage=%u num_planes=%u\n", __func__, -+ av_fourcc2str(fmt->pixelformat), fmt->width, fmt->height, fmt->plane_fmt[0].bytesperline, fmt->plane_fmt[0].sizeimage, fmt->num_planes); -+ } else { -+ struct v4l2_pix_format *fmt = &ctx->format.fmt.pix; -+ av_log(ctx, AV_LOG_DEBUG, "%s: pixelformat=%s width=%u height=%u bytesperline=%u sizeimage=%u\n", __func__, -+ av_fourcc2str(fmt->pixelformat), fmt->width, fmt->height, fmt->bytesperline, fmt->sizeimage); -+ } -+ -+ ret = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_DRM); -+ if (ret < 0) -+ goto fail; -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_STREAMON, &ctx->output_type); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: output stream on failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_STREAMON, &ctx->format.type); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: capture stream on failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ return 0; -+ -+fail: -+ ff_v4l2_request_uninit(avctx); -+ return ret; -+} -+ -+static int v4l2_request_probe_media_device(struct udev_device *device, AVCodecContext *avctx, uint32_t pixelformat, uint32_t buffersize, struct v4l2_ext_control *control, int count) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ int ret; -+ struct media_device_info device_info = {0}; -+ struct media_v2_topology topology = {0}; -+ struct media_v2_interface *interfaces = NULL; -+ struct udev *udev = udev_device_get_udev(device); -+ struct udev_device *video_device; -+ dev_t devnum; -+ -+ const char *path = udev_device_get_devnode(device); -+ if (!path) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get media device devnode failed\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ctx->media_fd = open(path, O_RDWR, 0); -+ if (ctx->media_fd < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: opening %s failed, %s (%d)\n", __func__, path, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = ioctl(ctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get media device info failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ av_log(avctx, AV_LOG_DEBUG, "%s: ctx=%p path=%s driver=%s\n", __func__, ctx, path, device_info.driver); -+ -+ ret = ioctl(ctx->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get media topology failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ if (topology.num_interfaces <= 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: media device has no interfaces\n", __func__); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ interfaces = av_mallocz(topology.num_interfaces * sizeof(struct media_v2_interface)); -+ if (!interfaces) { -+ av_log(avctx, AV_LOG_ERROR, "%s: allocating media interface struct failed\n", __func__); -+ ret = AVERROR(ENOMEM); -+ goto fail; -+ } -+ -+ topology.ptr_interfaces = (__u64)(uintptr_t)interfaces; -+ ret = ioctl(ctx->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology); -+ if (ret < 0) { -+ av_log(avctx, AV_LOG_ERROR, "%s: get media topology failed, %s (%d)\n", __func__, strerror(errno), errno); -+ ret = AVERROR(EINVAL); -+ goto fail; -+ } -+ -+ ret = AVERROR(EINVAL); -+ for (int i = 0; i < topology.num_interfaces; i++) { -+ if (interfaces[i].intf_type != MEDIA_INTF_T_V4L_VIDEO) -+ continue; -+ -+ devnum = makedev(interfaces[i].devnode.major, interfaces[i].devnode.minor); -+ video_device = udev_device_new_from_devnum(udev, 'c', devnum); -+ if (!video_device) { -+ av_log(avctx, AV_LOG_ERROR, "%s: video_device=%p\n", __func__, video_device); -+ continue; -+ } -+ -+ ret = v4l2_request_probe_video_device(video_device, avctx, pixelformat, buffersize, control, count); -+ udev_device_unref(video_device); -+ -+ if (!ret) { -+ av_freep(&interfaces); -+ av_log(avctx, AV_LOG_INFO, "Using V4L2 media device %s (%s) for %s\n", path, device_info.driver, av_fourcc2str(pixelformat)); -+ return 0; -+ } -+ } -+ -+fail: -+ av_freep(&interfaces); -+ if (ctx->media_fd >= 0) { -+ close(ctx->media_fd); -+ ctx->media_fd = -1; -+ } -+ return ret; -+} -+ -+int ff_v4l2_request_init(AVCodecContext *avctx, uint32_t pixelformat, uint32_t buffersize, struct v4l2_ext_control *control, int count) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ int ret = AVERROR(EINVAL); -+ struct udev *udev; -+ struct udev_enumerate *enumerate; -+ struct udev_list_entry *devices; -+ struct udev_list_entry *entry; -+ struct udev_device *device; -+ -+ av_log(avctx, AV_LOG_DEBUG, "%s: ctx=%p hw_device_ctx=%p hw_frames_ctx=%p\n", __func__, ctx, avctx->hw_device_ctx, avctx->hw_frames_ctx); -+ -+ ctx->av_class = &v4l2_request_context_class; -+ ctx->media_fd = -1; -+ ctx->video_fd = -1; -+ -+ udev = udev_new(); -+ if (!udev) { -+ av_log(avctx, AV_LOG_ERROR, "%s: allocating udev context failed\n", __func__); -+ ret = AVERROR(ENOMEM); -+ goto fail; -+ } -+ -+ enumerate = udev_enumerate_new(udev); -+ if (!enumerate) { -+ av_log(avctx, AV_LOG_ERROR, "%s: allocating udev enumerator failed\n", __func__); -+ ret = AVERROR(ENOMEM); -+ goto fail; -+ } -+ -+ udev_enumerate_add_match_subsystem(enumerate, "media"); -+ udev_enumerate_scan_devices(enumerate); -+ -+ devices = udev_enumerate_get_list_entry(enumerate); -+ udev_list_entry_foreach(entry, devices) { -+ const char *path = udev_list_entry_get_name(entry); -+ if (!path) -+ continue; -+ -+ device = udev_device_new_from_syspath(udev, path); -+ if (!device) -+ continue; -+ -+ ret = v4l2_request_probe_media_device(device, avctx, pixelformat, buffersize, control, count); -+ udev_device_unref(device); -+ -+ if (!ret) -+ break; -+ } -+ -+ udev_enumerate_unref(enumerate); -+ -+ if (!ret) -+ ret = v4l2_request_init_context(avctx); -+ else -+ av_log(avctx, AV_LOG_INFO, "No V4L2 media device found for %s\n", av_fourcc2str(pixelformat)); -+ -+fail: -+ udev_unref(udev); -+ return ret; -+} -+ -+int ff_v4l2_request_uninit(AVCodecContext *avctx) -+{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ int ret; -+ -+ av_log(ctx, AV_LOG_DEBUG, "%s: avctx=%p\n", __func__, avctx); -+ -+ if (ctx->video_fd >= 0) { -+ ret = ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type); -+ if (ret < 0) -+ av_log(avctx, AV_LOG_ERROR, "%s: output stream off failed, %s (%d)\n", __func__, strerror(errno), errno); -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->format.type); -+ if (ret < 0) -+ av_log(avctx, AV_LOG_ERROR, "%s: capture stream off failed, %s (%d)\n", __func__, strerror(errno), errno); -+ } -+ -+ if (avctx->hw_frames_ctx) { -+ AVHWFramesContext *hwfc = (AVHWFramesContext*)avctx->hw_frames_ctx->data; -+ av_buffer_pool_flush(hwfc->pool); -+ } -+ -+ if (ctx->video_fd >= 0) -+ close(ctx->video_fd); -+ -+ if (ctx->media_fd >= 0) -+ close(ctx->media_fd); -+ -+ return 0; -+} -+ -+static int v4l2_request_buffer_alloc(V4L2RequestContext *ctx, V4L2RequestBuffer *buf, enum v4l2_buf_type type) -+{ -+ int ret; + struct v4l2_plane planes[1] = {}; + struct v4l2_create_buffers buffers = { + .count = 1, @@ -1001,28 +639,18 @@ index 000000000000..595a53e6d4d8 + .format.type = type, + }; + -+ av_log(ctx, AV_LOG_DEBUG, "%s: buf=%p type=%u\n", __func__, buf, type); -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_G_FMT, &buffers.format); -+ if (ret < 0) { -+ av_log(ctx, AV_LOG_ERROR, "%s: get format failed for type %u, %s (%d)\n", __func__, type, strerror(errno), errno); -+ return ret; ++ // Get format details for the buffer to be created ++ if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &buffers.format) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to get format of type %d: %s (%d)\n", ++ type, strerror(errno), errno); ++ return AVERROR(errno); + } + -+ if (V4L2_TYPE_IS_MULTIPLANAR(buffers.format.type)) { -+ struct v4l2_pix_format_mplane *fmt = &buffers.format.fmt.pix_mp; -+ av_log(ctx, AV_LOG_DEBUG, "%s: pixelformat=%s width=%u height=%u bytesperline=%u sizeimage=%u num_planes=%u\n", __func__, -+ av_fourcc2str(fmt->pixelformat), fmt->width, fmt->height, fmt->plane_fmt[0].bytesperline, fmt->plane_fmt[0].sizeimage, fmt->num_planes); -+ } else { -+ struct v4l2_pix_format *fmt = &buffers.format.fmt.pix; -+ av_log(ctx, AV_LOG_DEBUG, "%s: pixelformat=%s width=%u height=%u bytesperline=%u sizeimage=%u\n", __func__, -+ av_fourcc2str(fmt->pixelformat), fmt->width, fmt->height, fmt->bytesperline, fmt->sizeimage); -+ } -+ -+ ret = ioctl(ctx->video_fd, VIDIOC_CREATE_BUFS, &buffers); -+ if (ret < 0) { -+ av_log(ctx, AV_LOG_ERROR, "%s: create buffers failed for type %u, %s (%d)\n", __func__, type, strerror(errno), errno); -+ return ret; ++ // Create the buffer ++ if (ioctl(ctx->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to create buffer of type %d: %s (%d)\n", ++ type, strerror(errno), errno); ++ return AVERROR(errno); + } + + if (V4L2_TYPE_IS_MULTIPLANAR(type)) { @@ -1045,23 +673,28 @@ index 000000000000..595a53e6d4d8 + buf->buffer.memory = V4L2_MEMORY_MMAP; + buf->buffer.index = buf->index; + -+ ret = ioctl(ctx->video_fd, VIDIOC_QUERYBUF, &buf->buffer); -+ if (ret < 0) { -+ av_log(ctx, AV_LOG_ERROR, "%s: query buffer %d failed, %s (%d)\n", __func__, buf->index, strerror(errno), errno); -+ return ret; ++ // Query more details of the created buffer ++ if (ioctl(ctx->video_fd, VIDIOC_QUERYBUF, &buf->buffer) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to query buffer %d of type %d: %s (%d)\n", ++ buf->index, type, strerror(errno), errno); ++ return AVERROR(errno); + } + -+ buf->buffer.timestamp.tv_usec = buf->index + 1; -+ ++ // Output buffers is mapped and capture buffers is exported + if (V4L2_TYPE_IS_OUTPUT(type)) { -+ uint32_t offset = V4L2_TYPE_IS_MULTIPLANAR(type) ? buf->buffer.m.planes[0].m.mem_offset : buf->buffer.m.offset; -+ void *addr = mmap(NULL, buf->size, PROT_READ | PROT_WRITE, MAP_SHARED, ctx->video_fd, offset); ++ off_t offset = V4L2_TYPE_IS_MULTIPLANAR(type) ? ++ buf->buffer.m.planes[0].m.mem_offset : ++ buf->buffer.m.offset; ++ void *addr = mmap(NULL, buf->size, PROT_READ | PROT_WRITE, MAP_SHARED, ++ ctx->video_fd, offset); + if (addr == MAP_FAILED) { -+ av_log(ctx, AV_LOG_ERROR, "%s: mmap failed, %s (%d)\n", __func__, strerror(errno), errno); -+ return -1; ++ av_log(ctx, AV_LOG_ERROR, "Failed to map output buffer %d: %s (%d)\n", ++ buf->index, strerror(errno), errno); ++ return AVERROR(errno); + } + -+ buf->addr = (uint8_t*)addr; ++ // Raw bitstream data is appended to output buffers ++ buf->addr = (uint8_t *)addr; + } else { + struct v4l2_exportbuffer exportbuffer = { + .type = type, @@ -1069,43 +702,46 @@ index 000000000000..595a53e6d4d8 + .flags = O_RDONLY, + }; + -+ ret = ioctl(ctx->video_fd, VIDIOC_EXPBUF, &exportbuffer); -+ if (ret < 0) { -+ av_log(ctx, AV_LOG_ERROR, "%s: export buffer %d failed, %s (%d)\n", __func__, buf->index, strerror(errno), errno); -+ return ret; ++ if (ioctl(ctx->video_fd, VIDIOC_EXPBUF, &exportbuffer) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to export capture buffer %d: %s (%d)\n", ++ buf->index, strerror(errno), errno); ++ return AVERROR(errno); + } + ++ // Used in the AVDRMFrameDescriptor for decoded frames + buf->fd = exportbuffer.fd; ++ ++ /* ++ * Use buffer index as base for V4L2 frame reference. ++ * This works because a capture buffer is closely tied to a AVFrame ++ * and FFmpeg handle all frame reference tracking for us. ++ */ ++ buf->buffer.timestamp.tv_usec = buf->index + 1; + } + -+ av_log(ctx, AV_LOG_DEBUG, "%s: buf=%p index=%d fd=%d addr=%p width=%u height=%u size=%u\n", __func__, -+ buf, buf->index, buf->fd, buf->addr, buf->width, buf->height, buf->size); + return 0; +} + +static void v4l2_request_buffer_free(V4L2RequestBuffer *buf) +{ -+ av_log(NULL, AV_LOG_DEBUG, "%s: buf=%p index=%d fd=%d addr=%p width=%u height=%u size=%u\n", __func__, -+ buf, buf->index, buf->fd, buf->addr, buf->width, buf->height, buf->size); -+ -+ if (buf->addr) ++ // Unmap output buffers ++ if (buf->addr) { + munmap(buf->addr, buf->size); ++ buf->addr = NULL; ++ } + -+ if (buf->fd >= 0) ++ // Close exported capture buffers or requests for output buffers ++ if (buf->fd >= 0) { + close(buf->fd); ++ buf->fd = -1; ++ } +} + +static void v4l2_request_frame_free(void *opaque, uint8_t *data) +{ -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)data; ++ V4L2RequestFrameDescriptor *desc = (V4L2RequestFrameDescriptor *)data; + -+ av_log(NULL, AV_LOG_DEBUG, "%s: opaque=%p data=%p request_fd=%d\n", __func__, opaque, data, req->request_fd); -+ -+ if (req->request_fd >= 0) -+ close(req->request_fd); -+ -+ v4l2_request_buffer_free(&req->capture); -+ v4l2_request_buffer_free(&req->output); ++ v4l2_request_buffer_free(&desc->capture); + + av_free(data); +} @@ -1113,91 +749,59 @@ index 000000000000..595a53e6d4d8 +static AVBufferRef *v4l2_request_frame_alloc(void *opaque, size_t size) +{ + V4L2RequestContext *ctx = opaque; -+ V4L2RequestDescriptor *req; ++ V4L2RequestFrameDescriptor *desc; + AVBufferRef *ref; + uint8_t *data; -+ int ret; + + data = av_mallocz(size); + if (!data) + return NULL; + -+ av_log(ctx, AV_LOG_DEBUG, "%s: size=%zu data=%p\n", __func__, size, data); -+ + ref = av_buffer_create(data, size, v4l2_request_frame_free, ctx, 0); + if (!ref) { -+ av_freep(&data); ++ av_free(data); + return NULL; + } + -+ req = (V4L2RequestDescriptor*)data; -+ req->request_fd = -1; -+ req->output.fd = -1; -+ req->capture.fd = -1; ++ desc = (V4L2RequestFrameDescriptor *)data; ++ desc->capture.fd = -1; + -+ ret = v4l2_request_buffer_alloc(ctx, &req->output, ctx->output_type); -+ if (ret < 0) { ++ // Create a V4L2 capture buffer for this AVFrame ++ if (v4l2_request_buffer_alloc(ctx, &desc->capture, ctx->format.type) < 0) { + av_buffer_unref(&ref); + return NULL; + } + -+ ret = v4l2_request_buffer_alloc(ctx, &req->capture, ctx->format.type); -+ if (ret < 0) { -+ av_buffer_unref(&ref); -+ return NULL; -+ } ++ // TODO: Set AVDRMFrameDescriptor of this AVFrame + -+ ret = ioctl(ctx->media_fd, MEDIA_IOC_REQUEST_ALLOC, &req->request_fd); -+ if (ret < 0) { -+ av_log(ctx, AV_LOG_ERROR, "%s: request alloc failed, %s (%d)\n", __func__, strerror(errno), errno); -+ av_buffer_unref(&ref); -+ return NULL; -+ } -+ -+ av_log(ctx, AV_LOG_DEBUG, "%s: size=%zu data=%p request_fd=%d\n", __func__, size, data, req->request_fd); + return ref; +} + -+static void v4l2_request_pool_free(void *opaque) -+{ -+ av_log(NULL, AV_LOG_DEBUG, "%s: opaque=%p\n", __func__, opaque); -+} -+ +static void v4l2_request_hwframe_ctx_free(AVHWFramesContext *hwfc) +{ -+ av_log(NULL, AV_LOG_DEBUG, "%s: hwfc=%p pool=%p\n", __func__, hwfc, hwfc->pool); -+ -+ av_buffer_pool_flush(hwfc->pool); + av_buffer_pool_uninit(&hwfc->pool); +} + -+int ff_v4l2_request_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx) ++int ff_v4l2_request_frame_params(AVCodecContext *avctx, ++ AVBufferRef *hw_frames_ctx) +{ -+ V4L2RequestContext *ctx = avctx->internal->hwaccel_priv_data; -+ AVHWFramesContext *hwfc = (AVHWFramesContext*)hw_frames_ctx->data; -+ uint32_t pixelformat; ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ AVHWFramesContext *hwfc = (AVHWFramesContext *)hw_frames_ctx->data; + + hwfc->format = AV_PIX_FMT_DRM_PRIME; ++ // TODO: Set sw_format based on capture buffer format + hwfc->sw_format = AV_PIX_FMT_NONE; + + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->format.type)) { + hwfc->width = ctx->format.fmt.pix_mp.width; + hwfc->height = ctx->format.fmt.pix_mp.height; -+ pixelformat = ctx->format.fmt.pix_mp.pixelformat; + } else { + hwfc->width = ctx->format.fmt.pix.width; + hwfc->height = ctx->format.fmt.pix.height; -+ pixelformat = ctx->format.fmt.pix.pixelformat; + } + -+ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { -+ if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) { -+ hwfc->sw_format = v4l2_request_capture_pixelformats[i].sw_format; -+ break; -+ } -+ } -+ -+ hwfc->pool = av_buffer_pool_init2(sizeof(V4L2RequestDescriptor), ctx, v4l2_request_frame_alloc, v4l2_request_pool_free); ++ hwfc->pool = av_buffer_pool_init2(sizeof(V4L2RequestFrameDescriptor), ctx, ++ v4l2_request_frame_alloc, NULL); + if (!hwfc->pool) + return AVERROR(ENOMEM); + @@ -1216,16 +820,159 @@ index 000000000000..595a53e6d4d8 + hwfc->initial_pool_size += 2; + } + -+ av_log(ctx, AV_LOG_DEBUG, "%s: avctx=%p hw_frames_ctx=%p hwfc=%p pool=%p width=%d height=%d initial_pool_size=%d\n", __func__, -+ avctx, hw_frames_ctx, hwfc, hwfc->pool, hwfc->width, hwfc->height, hwfc->initial_pool_size); + return 0; +} ++ ++int ff_v4l2_request_uninit(AVCodecContext *avctx) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ ++ if (ctx->video_fd >= 0) { ++ // TODO: Flush and wait on all pending requests ++ ++ // Stop output queue ++ if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) { ++ av_log(ctx, AV_LOG_WARNING, "Failed to stop output streaming: %s (%d)\n", ++ strerror(errno), errno); ++ } ++ ++ // Stop capture queue ++ if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->format.type) < 0) { ++ av_log(ctx, AV_LOG_WARNING, "Failed to stop capture streaming: %s (%d)\n", ++ strerror(errno), errno); ++ } ++ ++ // Release output buffers ++ for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) { ++ v4l2_request_buffer_free(&ctx->output[i]); ++ } ++ ++ // Close video device file descriptor ++ close(ctx->video_fd); ++ ctx->video_fd = -1; ++ } ++ ++ // Ownership of media device file descriptor may belong to hwdevice ++ if (ctx->device_ref) { ++ av_buffer_unref(&ctx->device_ref); ++ ctx->media_fd = -1; ++ } else if (ctx->media_fd >= 0) { ++ close(ctx->media_fd); ++ ctx->media_fd = -1; ++ } ++ ++ ff_mutex_destroy(&ctx->mutex); ++ ++ return 0; ++} ++ ++static int v4l2_request_init_context(AVCodecContext *avctx) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ int ret; ++ ++ // Initialize context state ++ ff_mutex_init(&ctx->mutex, NULL); ++ for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) { ++ ctx->output[i].index = i; ++ ctx->output[i].fd = -1; ++ } ++ atomic_init(&ctx->next_output, 0); ++ atomic_init(&ctx->queued_output, 0); ++ atomic_init(&ctx->queued_request, 0); ++ atomic_init(&ctx->queued_capture, 0); ++ ++ // Get format details for capture buffers ++ if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &ctx->format) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to get capture format: %s (%d)\n", ++ strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ // Create frame context and allocate initial capture buffers ++ ret = ff_decode_get_hw_frames_ctx(avctx, AV_HWDEVICE_TYPE_V4L2REQUEST); ++ if (ret < 0) ++ goto fail; ++ ++ // Allocate output buffers for circular queue ++ for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) { ++ ret = v4l2_request_buffer_alloc(ctx, &ctx->output[i], ctx->output_type); ++ if (ret < 0) ++ goto fail; ++ } ++ ++ // Allocate requests for circular queue ++ for (int i = 0; i < FF_ARRAY_ELEMS(ctx->output); i++) { ++ if (ioctl(ctx->media_fd, MEDIA_IOC_REQUEST_ALLOC, &ctx->output[i].fd) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to allocate request %d: %s (%d)\n", ++ i, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ } ++ ++ // Start output queue ++ if (ioctl(ctx->video_fd, VIDIOC_STREAMON, &ctx->output_type) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to start output streaming: %s (%d)\n", ++ strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ // Start capture queue ++ if (ioctl(ctx->video_fd, VIDIOC_STREAMON, &ctx->format.type) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to start capture streaming: %s (%d)\n", ++ strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ return 0; ++ ++fail: ++ ff_v4l2_request_uninit(avctx); ++ return ret; ++} ++ ++int ff_v4l2_request_init(AVCodecContext *avctx, ++ uint32_t pixelformat, uint32_t buffersize, ++ struct v4l2_ext_control *control, int count) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ AVV4L2RequestDeviceContext *hwctx = NULL; ++ ++ // Set initial default values ++ ctx->av_class = &v4l2_request_context_class; ++ ctx->media_fd = -1; ++ ctx->video_fd = -1; ++ ++ // Try to use media device file descriptor opened and owned by hwdevice ++ if (avctx->hw_device_ctx) { ++ AVHWDeviceContext *device_ctx = (AVHWDeviceContext *)avctx->hw_device_ctx->data; ++ if (device_ctx->type == AV_HWDEVICE_TYPE_V4L2REQUEST && device_ctx->hwctx) { ++ hwctx = device_ctx->hwctx; ++ ctx->media_fd = hwctx->media_fd; ++ } ++ } ++ ++ // TODO: Probe for a capable media and video device ++ ++ // Transfer (or return) ownership of media device file descriptor to hwdevice ++ if (hwctx) { ++ hwctx->media_fd = ctx->media_fd; ++ ctx->device_ref = av_buffer_ref(avctx->hw_device_ctx); ++ } ++ ++ // Create buffers and finalize init ++ return v4l2_request_init_context(avctx); ++} diff --git a/libavcodec/v4l2_request.h b/libavcodec/v4l2_request.h new file mode 100644 -index 000000000000..5f105d8de1fe +index 0000000000..621caaf28c --- /dev/null +++ b/libavcodec/v4l2_request.h -@@ -0,0 +1,78 @@ +@@ -0,0 +1,84 @@ +/* + * This file is part of FFmpeg. + * @@ -1247,18 +994,13 @@ index 000000000000..5f105d8de1fe +#ifndef AVCODEC_V4L2_REQUEST_H +#define AVCODEC_V4L2_REQUEST_H + ++#include ++#include ++ +#include + +#include "libavutil/hwcontext_drm.h" -+ -+typedef struct V4L2RequestContext { -+ const AVClass *av_class; -+ int video_fd; -+ int media_fd; -+ enum v4l2_buf_type output_type; -+ struct v4l2_format format; -+ int timestamp; -+} V4L2RequestContext; ++#include "libavutil/thread.h" + +typedef struct V4L2RequestBuffer { + int index; @@ -1272,220 +1014,55 @@ index 000000000000..5f105d8de1fe + struct v4l2_buffer buffer; +} V4L2RequestBuffer; + -+typedef struct V4L2RequestDescriptor { -+ AVDRMFrameDescriptor drm; -+ int request_fd; -+ V4L2RequestBuffer output; -+ V4L2RequestBuffer capture; -+} V4L2RequestDescriptor; ++typedef struct V4L2RequestContext { ++ const AVClass *av_class; ++ AVBufferRef *device_ref; ++ int media_fd; ++ int video_fd; ++ struct v4l2_format format; ++ enum v4l2_buf_type output_type; ++ AVMutex mutex; ++ V4L2RequestBuffer output[4]; ++ atomic_int_least8_t next_output; ++ atomic_uint_least32_t queued_output; ++ atomic_uint_least32_t queued_request; ++ atomic_uint_least64_t queued_capture; ++ int (*post_probe)(AVCodecContext *avctx); ++} V4L2RequestContext; + -+uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame); ++typedef struct V4L2RequestPictureContext { ++ V4L2RequestBuffer *output; ++ V4L2RequestBuffer *capture; ++} V4L2RequestPictureContext; + -+int ff_v4l2_request_reset_frame(AVCodecContext *avctx, AVFrame *frame); ++int ff_v4l2_request_query_control(AVCodecContext *avctx, ++ struct v4l2_query_ext_ctrl *control); + -+int ff_v4l2_request_append_output_buffer(AVCodecContext *avctx, AVFrame *frame, const uint8_t *data, uint32_t size); ++int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx, ++ uint32_t id); + -+int ff_v4l2_request_set_controls(AVCodecContext *avctx, struct v4l2_ext_control *control, int count); ++int ff_v4l2_request_set_request_controls(V4L2RequestContext *ctx, int request_fd, ++ struct v4l2_ext_control *control, int count); + -+int ff_v4l2_request_get_controls(AVCodecContext *avctx, struct v4l2_ext_control *control, int count); ++int ff_v4l2_request_set_controls(AVCodecContext *avctx, ++ struct v4l2_ext_control *control, int count); + -+int ff_v4l2_request_query_control(AVCodecContext *avctx, struct v4l2_query_ext_ctrl *control); -+ -+int ff_v4l2_request_query_control_default_value(AVCodecContext *avctx, uint32_t id); -+ -+int ff_v4l2_request_decode_slice(AVCodecContext *avctx, AVFrame *frame, struct v4l2_ext_control *control, int count, int first_slice, int last_slice); -+ -+int ff_v4l2_request_decode_frame(AVCodecContext *avctx, AVFrame *frame, struct v4l2_ext_control *control, int count); -+ -+int ff_v4l2_request_init(AVCodecContext *avctx, uint32_t pixelformat, uint32_t buffersize, struct v4l2_ext_control *control, int count); ++int ff_v4l2_request_frame_params(AVCodecContext *avctx, ++ AVBufferRef *hw_frames_ctx); + +int ff_v4l2_request_uninit(AVCodecContext *avctx); + -+int ff_v4l2_request_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx); ++int ff_v4l2_request_init(AVCodecContext *avctx, ++ uint32_t pixelformat, uint32_t buffersize, ++ struct v4l2_ext_control *control, int count); + +#endif /* AVCODEC_V4L2_REQUEST_H */ - -From b97ddcb4b2476f8d82dc7f32fe8c7e35f50d0234 Mon Sep 17 00:00:00 2001 -From: Boris Brezillon -Date: Wed, 22 May 2019 14:44:22 +0200 -Subject: [PATCH 03/11] h264dec: add ref_pic_marking and pic_order_cnt bit_size - to slice context - -Used by V4L2 request API h264 hwaccel - -Signed-off-by: Boris Brezillon -Signed-off-by: Jonas Karlman ---- - libavcodec/h264_slice.c | 6 +++++- - libavcodec/h264dec.h | 2 ++ - 2 files changed, 7 insertions(+), 1 deletion(-) - -diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c -index 7767e16cf15c..e782a6920030 100644 ---- a/libavcodec/h264_slice.c -+++ b/libavcodec/h264_slice.c -@@ -1670,7 +1670,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, - unsigned int slice_type, tmp, i; - int field_pic_flag, bottom_field_flag; - int first_slice = sl == h->slice_ctx && !h->current_slice; -- int picture_structure; -+ int picture_structure, pos; - - if (first_slice) - av_assert0(!h->setup_finished); -@@ -1761,6 +1761,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, - - sl->poc_lsb = 0; - sl->delta_poc_bottom = 0; -+ pos = sl->gb.index; - if (sps->poc_type == 0) { - sl->poc_lsb = get_bits(&sl->gb, sps->log2_max_poc_lsb); - -@@ -1775,6 +1776,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, - if (pps->pic_order_present == 1 && picture_structure == PICT_FRAME) - sl->delta_poc[1] = get_se_golomb(&sl->gb); - } -+ sl->pic_order_cnt_bit_size = sl->gb.index - pos; - - sl->redundant_pic_count = 0; - if (pps->redundant_pic_cnt_present) -@@ -1814,9 +1816,11 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, - - sl->explicit_ref_marking = 0; - if (nal->ref_idc) { -+ pos = sl->gb.index; - ret = ff_h264_decode_ref_pic_marking(sl, &sl->gb, nal, h->avctx); - if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) - return AVERROR_INVALIDDATA; -+ sl->ref_pic_marking_bit_size = sl->gb.index - pos; - } - - if (sl->slice_type_nos != AV_PICTURE_TYPE_I && pps->cabac) { -diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h -index 9a1ec1bacec9..a87415f822f7 100644 ---- a/libavcodec/h264dec.h -+++ b/libavcodec/h264dec.h -@@ -314,6 +314,7 @@ typedef struct H264SliceContext { - MMCO mmco[H264_MAX_MMCO_COUNT]; - int nb_mmco; - int explicit_ref_marking; -+ int ref_pic_marking_bit_size; - - int frame_num; - int idr_pic_id; -@@ -322,6 +323,7 @@ typedef struct H264SliceContext { - int delta_poc[2]; - int curr_pic_num; - int max_pic_num; -+ int pic_order_cnt_bit_size; - } H264SliceContext; - - /** - -From 9142ddbf08edbbbf7c2c223b3bf3827344ac905c Mon Sep 17 00:00:00 2001 -From: Jernej Skrabec -Date: Sat, 15 Dec 2018 22:32:16 +0100 -Subject: [PATCH 04/11] Add V4L2 request API h264 hwaccel - -Signed-off-by: Jernej Skrabec -Signed-off-by: Jonas Karlman ---- - configure | 3 + - libavcodec/Makefile | 1 + - libavcodec/h264_slice.c | 4 + - libavcodec/h264dec.c | 3 + - libavcodec/hwaccels.h | 1 + - libavcodec/v4l2_request_h264.c | 461 +++++++++++++++++++++++++++++++++ - 6 files changed, 473 insertions(+) - create mode 100644 libavcodec/v4l2_request_h264.c - -diff --git a/configure b/configure -index cbc5442c8a0e..8a475352c2ca 100755 ---- a/configure -+++ b/configure -@@ -3033,6 +3033,8 @@ h264_dxva2_hwaccel_deps="dxva2" - h264_dxva2_hwaccel_select="h264_decoder" - h264_nvdec_hwaccel_deps="nvdec" - h264_nvdec_hwaccel_select="h264_decoder" -+h264_v4l2request_hwaccel_deps="v4l2_request h264_v4l2_request" -+h264_v4l2request_hwaccel_select="h264_decoder" - h264_vaapi_hwaccel_deps="vaapi" - h264_vaapi_hwaccel_select="h264_decoder" - h264_vdpau_hwaccel_deps="vdpau" -@@ -6887,6 +6889,7 @@ if enabled v4l2_m2m; then - fi - - check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns -+check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_H264_SLICE;" - - check_headers sys/videoio.h - test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete -diff --git a/libavcodec/Makefile b/libavcodec/Makefile -index 0148242ed0c3..26004171b31a 100644 ---- a/libavcodec/Makefile -+++ b/libavcodec/Makefile -@@ -990,6 +990,7 @@ OBJS-$(CONFIG_H264_D3D11VA_HWACCEL) += dxva2_h264.o - OBJS-$(CONFIG_H264_DXVA2_HWACCEL) += dxva2_h264.o - OBJS-$(CONFIG_H264_NVDEC_HWACCEL) += nvdec_h264.o - OBJS-$(CONFIG_H264_QSV_HWACCEL) += qsvdec.o -+OBJS-$(CONFIG_H264_V4L2REQUEST_HWACCEL) += v4l2_request_h264.o - OBJS-$(CONFIG_H264_VAAPI_HWACCEL) += vaapi_h264.o - OBJS-$(CONFIG_H264_VDPAU_HWACCEL) += vdpau_h264.o - OBJS-$(CONFIG_H264_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o -diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c -index e782a6920030..3d0d45b2a3f5 100644 ---- a/libavcodec/h264_slice.c -+++ b/libavcodec/h264_slice.c -@@ -778,6 +778,7 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) - #define HWACCEL_MAX (CONFIG_H264_DXVA2_HWACCEL + \ - (CONFIG_H264_D3D11VA_HWACCEL * 2) + \ - CONFIG_H264_NVDEC_HWACCEL + \ -+ CONFIG_H264_V4L2REQUEST_HWACCEL + \ - CONFIG_H264_VAAPI_HWACCEL + \ - CONFIG_H264_VIDEOTOOLBOX_HWACCEL + \ - CONFIG_H264_VDPAU_HWACCEL) -@@ -867,6 +868,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) - #endif - #if CONFIG_H264_VAAPI_HWACCEL - *fmt++ = AV_PIX_FMT_VAAPI; -+#endif -+#if CONFIG_H264_V4L2REQUEST_HWACCEL -+ *fmt++ = AV_PIX_FMT_DRM_PRIME; - #endif - if (h->avctx->codec->pix_fmts) - choices = h->avctx->codec->pix_fmts; -diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c -index 2d691731c5d5..33cc25e1d335 100644 ---- a/libavcodec/h264dec.c -+++ b/libavcodec/h264dec.c -@@ -1093,6 +1093,9 @@ const FFCodec ff_h264_decoder = { - #endif - #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL - HWACCEL_VIDEOTOOLBOX(h264), -+#endif -+#if CONFIG_H264_V4L2REQUEST_HWACCEL -+ HWACCEL_V4L2REQUEST(h264), - #endif - NULL - }, -diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h -index aca55831f32f..014b95f0c05d 100644 ---- a/libavcodec/hwaccels.h -+++ b/libavcodec/hwaccels.h -@@ -33,6 +33,7 @@ extern const AVHWAccel ff_h264_d3d11va_hwaccel; - extern const AVHWAccel ff_h264_d3d11va2_hwaccel; - extern const AVHWAccel ff_h264_dxva2_hwaccel; - extern const AVHWAccel ff_h264_nvdec_hwaccel; -+extern const AVHWAccel ff_h264_v4l2request_hwaccel; - extern const AVHWAccel ff_h264_vaapi_hwaccel; - extern const AVHWAccel ff_h264_vdpau_hwaccel; - extern const AVHWAccel ff_h264_videotoolbox_hwaccel; -diff --git a/libavcodec/v4l2_request_h264.c b/libavcodec/v4l2_request_h264.c +diff --git a/libavcodec/v4l2_request_internal.h b/libavcodec/v4l2_request_internal.h new file mode 100644 -index 000000000000..5c2bea17820d +index 0000000000..554b663ab4 --- /dev/null -+++ b/libavcodec/v4l2_request_h264.c -@@ -0,0 +1,461 @@ ++++ b/libavcodec/v4l2_request_internal.h +@@ -0,0 +1,42 @@ +/* + * This file is part of FFmpeg. + * @@ -1504,470 +1081,1457 @@ index 000000000000..5c2bea17820d + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + -+#include "h264dec.h" -+#include "hwconfig.h" ++#ifndef AVCODEC_V4L2_REQUEST_INTERNAL_H ++#define AVCODEC_V4L2_REQUEST_INTERNAL_H ++ ++#include ++ +#include "internal.h" +#include "v4l2_request.h" + -+typedef struct V4L2RequestControlsH264 { -+ struct v4l2_ctrl_h264_sps sps; -+ struct v4l2_ctrl_h264_pps pps; -+ struct v4l2_ctrl_h264_scaling_matrix scaling_matrix; -+ struct v4l2_ctrl_h264_decode_params decode_params; -+ struct v4l2_ctrl_h264_slice_params slice_params; -+ struct v4l2_ctrl_h264_pred_weights pred_weights; -+ int pred_weights_required; -+ int first_slice; -+ int num_slices; -+} V4L2RequestControlsH264; ++typedef struct V4L2RequestFrameDescriptor { ++ AVDRMFrameDescriptor base; ++ V4L2RequestBuffer capture; ++} V4L2RequestFrameDescriptor; + -+typedef struct V4L2RequestContextH264 { -+ V4L2RequestContext base; -+ int decode_mode; -+ int start_code; -+} V4L2RequestContextH264; -+ -+static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 }; -+ -+static void fill_weight_factors(struct v4l2_h264_weight_factors *factors, int list, const H264SliceContext *sl) ++static inline V4L2RequestContext *v4l2_request_context(AVCodecContext *avctx) +{ -+ for (int i = 0; i < sl->ref_count[list]; i++) { -+ if (sl->pwt.luma_weight_flag[list]) { -+ factors->luma_weight[i] = sl->pwt.luma_weight[i][list][0]; -+ factors->luma_offset[i] = sl->pwt.luma_weight[i][list][1]; -+ } else { -+ factors->luma_weight[i] = 1 << sl->pwt.luma_log2_weight_denom; -+ factors->luma_offset[i] = 0; -+ } -+ for (int j = 0; j < 2; j++) { -+ if (sl->pwt.chroma_weight_flag[list]) { -+ factors->chroma_weight[i][j] = sl->pwt.chroma_weight[i][list][j][0]; -+ factors->chroma_offset[i][j] = sl->pwt.chroma_weight[i][list][j][1]; -+ } else { -+ factors->chroma_weight[i][j] = 1 << sl->pwt.chroma_log2_weight_denom; -+ factors->chroma_offset[i][j] = 0; -+ } -+ } -+ } ++ return (V4L2RequestContext *)avctx->internal->hwaccel_priv_data; +} + -+static void fill_dpb_entry(struct v4l2_h264_dpb_entry *entry, const H264Picture *pic, int long_idx) ++static inline V4L2RequestFrameDescriptor *v4l2_request_framedesc(AVFrame *frame) +{ -+ entry->reference_ts = ff_v4l2_request_get_capture_timestamp(pic->f); -+ entry->pic_num = pic->pic_id; -+ entry->frame_num = pic->long_ref ? long_idx : pic->frame_num; -+ entry->fields = pic->reference & V4L2_H264_FRAME_REF; -+ entry->flags = V4L2_H264_DPB_ENTRY_FLAG_VALID; -+ if (entry->fields) -+ entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; -+ if (pic->long_ref) -+ entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM; -+ if (pic->field_picture) -+ entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD; -+ if (pic->field_poc[0] != INT_MAX) -+ entry->top_field_order_cnt = pic->field_poc[0]; -+ if (pic->field_poc[1] != INT_MAX) -+ entry->bottom_field_order_cnt = pic->field_poc[1]; ++ return (V4L2RequestFrameDescriptor *)frame->data[0]; +} + -+static void fill_dpb(struct v4l2_ctrl_h264_decode_params *decode, const H264Context *h) -+{ -+ int entries = 0; -+ -+ for (int i = 0; i < h->short_ref_count; i++) { -+ const H264Picture *pic = h->short_ref[i]; -+ if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) -+ fill_dpb_entry(&decode->dpb[entries++], pic, pic->pic_id); ++#endif /* AVCODEC_V4L2_REQUEST_INTERNAL_H */ + +From f2779c95d7416be33aa3b50f1c7f1f33f5b9aad5 Mon Sep 17 00:00:00 2001 +From: Jonas Karlman +Date: Tue, 6 Aug 2024 09:06:02 +0000 +Subject: [PATCH 03/11] avcodec/v4l2request: Probe for a capable media and + video device + +Probe all media devices and its linked video devices to locate a video +device that support stateless decoding of the specific codec using the +V4L2 Request API. + +When AVV4L2RequestDeviceContext.media_fd is a valid file descriptor only +video devices for the opened media device is probed. +E.g. using a -init_hw_device v4l2request:/dev/media1 parameter. + +A video device is deemed capable when all tests pass, e.g. kernel driver +support the codec, FFmpeg v4l2-request code know about the buffer format +and kernel driver report that the frame size is supported. + +A codec specific hook, post_probe, is supported to e.g. test for codec +specific limitations, PROFILE and LEVEL controls. + +Basic flow for initialization follow the kernel Memory-to-memory +Stateless Video Decoder Interface > Initialization [1]. + +[1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#initialization + +Co-developed-by: Jernej Skrabec +Signed-off-by: Jernej Skrabec +Co-developed-by: Alex Bee +Signed-off-by: Alex Bee +Signed-off-by: Jonas Karlman +--- + configure | 4 +- + libavcodec/Makefile | 2 +- + libavcodec/v4l2_request.c | 18 +- + libavcodec/v4l2_request_internal.h | 9 + + libavcodec/v4l2_request_probe.c | 614 +++++++++++++++++++++++++++++ + 5 files changed, 641 insertions(+), 6 deletions(-) + create mode 100644 libavcodec/v4l2_request_probe.c + +diff --git a/configure b/configure +index b8f82f0fbf..98a342d109 100755 +--- a/configure ++++ b/configure +@@ -1966,6 +1966,7 @@ EXTERNAL_LIBRARY_LIST=" + libtorch + libtwolame + libuavs3d ++ libudev + libv4l2 + libvmaf + libvorbis +@@ -3148,7 +3149,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" + ffnvcodec_deps_any="libdl LoadLibrary" + mediacodec_deps="android mediandk" + nvdec_deps="ffnvcodec" +-v4l2_request_deps="linux_media_h v4l2_timeval_to_ns" ++v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm libudev" + vaapi_x11_deps="xlib_x11" + videotoolbox_hwaccel_deps="videotoolbox pthreads" + videotoolbox_hwaccel_extralibs="-framework QuartzCore" +@@ -7202,6 +7203,7 @@ fi + + if enabled v4l2_request; then + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns ++ check_pkg_config libudev libudev libudev.h udev_new + fi + + check_headers sys/videoio.h +diff --git a/libavcodec/Makefile b/libavcodec/Makefile +index af60923827..ebd2dc54b9 100644 +--- a/libavcodec/Makefile ++++ b/libavcodec/Makefile +@@ -176,7 +176,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o + OBJS-$(CONFIG_VP56DSP) += vp56dsp.o + OBJS-$(CONFIG_VP8DSP) += vp8dsp.o + OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o +-OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o ++OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o + OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o + OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o + +diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c +index 436c8de4a4..6f44c5d826 100644 +--- a/libavcodec/v4l2_request.c ++++ b/libavcodec/v4l2_request.c +@@ -244,7 +244,11 @@ static AVBufferRef *v4l2_request_frame_alloc(void *opaque, size_t size) + return NULL; + } + +- // TODO: Set AVDRMFrameDescriptor of this AVFrame ++ // Set AVDRMFrameDescriptor of this AVFrame ++ if (ff_v4l2_request_set_drm_descriptor(desc, &ctx->format) < 0) { ++ av_buffer_unref(&ref); ++ return NULL; + } -+ -+ if (!h->long_ref_count) -+ return; -+ -+ for (int i = 0; i < FF_ARRAY_ELEMS(h->long_ref); i++) { -+ const H264Picture *pic = h->long_ref[i]; -+ if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) -+ fill_dpb_entry(&decode->dpb[entries++], pic, i); -+ } -+} -+ -+static void fill_ref_list(struct v4l2_h264_reference *reference, struct v4l2_ctrl_h264_decode_params *decode, const H264Ref *ref) -+{ -+ uint64_t timestamp; -+ -+ if (!ref->parent) -+ return; -+ -+ timestamp = ff_v4l2_request_get_capture_timestamp(ref->parent->f); -+ -+ for (uint8_t i = 0; i < FF_ARRAY_ELEMS(decode->dpb); i++) { -+ struct v4l2_h264_dpb_entry *entry = &decode->dpb[i]; -+ if ((entry->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) && -+ entry->reference_ts == timestamp) { -+ reference->fields = ref->reference & V4L2_H264_FRAME_REF; -+ reference->index = i; -+ return; -+ } -+ } -+} -+ -+static void fill_sps(struct v4l2_ctrl_h264_sps *ctrl, const H264Context *h) -+{ -+ const SPS *sps = h->ps.sps; -+ -+ *ctrl = (struct v4l2_ctrl_h264_sps) { -+ .profile_idc = sps->profile_idc, -+ .constraint_set_flags = sps->constraint_set_flags, -+ .level_idc = sps->level_idc, -+ .seq_parameter_set_id = sps->sps_id, -+ .chroma_format_idc = sps->chroma_format_idc, -+ .bit_depth_luma_minus8 = sps->bit_depth_luma - 8, -+ .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8, -+ .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4, -+ .pic_order_cnt_type = sps->poc_type, -+ .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4, -+ .max_num_ref_frames = sps->ref_frame_count, -+ .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length, -+ .offset_for_non_ref_pic = sps->offset_for_non_ref_pic, -+ .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field, -+ .pic_width_in_mbs_minus1 = h->mb_width - 1, -+ .pic_height_in_map_units_minus1 = sps->frame_mbs_only_flag ? h->mb_height - 1 : h->mb_height / 2 - 1, -+ }; -+ -+ if (sps->poc_cycle_length > 0 && sps->poc_cycle_length <= 255) -+ memcpy(ctrl->offset_for_ref_frame, sps->offset_for_ref_frame, sps->poc_cycle_length * sizeof(ctrl->offset_for_ref_frame[0])); -+ -+ if (sps->residual_color_transform_flag) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; -+ if (sps->transform_bypass) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; -+ if (sps->delta_pic_order_always_zero_flag) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO; -+ if (sps->gaps_in_frame_num_allowed_flag) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED; -+ if (sps->frame_mbs_only_flag) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY; -+ if (sps->mb_aff) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; -+ if (sps->direct_8x8_inference_flag) -+ ctrl->flags |= V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE; -+} -+ -+static void fill_pps(struct v4l2_ctrl_h264_pps *ctrl, const H264Context *h) -+{ -+ const SPS *sps = h->ps.sps; -+ const PPS *pps = h->ps.pps; -+ const H264SliceContext *sl = &h->slice_ctx[0]; -+ int qp_bd_offset = 6 * (sps->bit_depth_luma - 8); -+ -+ *ctrl = (struct v4l2_ctrl_h264_pps) { -+ .pic_parameter_set_id = sl->pps_id, -+ .seq_parameter_set_id = pps->sps_id, -+ .num_slice_groups_minus1 = pps->slice_group_count - 1, -+ .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1, -+ .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1, -+ .weighted_bipred_idc = pps->weighted_bipred_idc, -+ .pic_init_qp_minus26 = pps->init_qp - 26 - qp_bd_offset, -+ .pic_init_qs_minus26 = pps->init_qs - 26 - qp_bd_offset, -+ .chroma_qp_index_offset = pps->chroma_qp_index_offset[0], -+ .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1], -+ }; -+ -+ if (pps->cabac) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; -+ if (pps->pic_order_present) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT; -+ if (pps->weighted_pred) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; -+ if (pps->deblocking_filter_parameters_present) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; -+ if (pps->constrained_intra_pred) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED; -+ if (pps->redundant_pic_cnt_present) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; -+ if (pps->transform_8x8_mode) -+ ctrl->flags |= V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE; -+ -+ /* FFmpeg always provide a scaling matrix */ -+ ctrl->flags |= V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT; -+} -+ -+static int v4l2_request_h264_start_frame(AVCodecContext *avctx, -+ av_unused const uint8_t *buffer, -+ av_unused uint32_t size) -+{ -+ const H264Context *h = avctx->priv_data; -+ const PPS *pps = h->ps.pps; -+ const SPS *sps = h->ps.sps; -+ const H264SliceContext *sl = &h->slice_ctx[0]; -+ V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; -+ -+ fill_sps(&controls->sps, h); -+ fill_pps(&controls->pps, h); -+ -+ memcpy(controls->scaling_matrix.scaling_list_4x4, pps->scaling_matrix4, sizeof(controls->scaling_matrix.scaling_list_4x4)); -+ memcpy(controls->scaling_matrix.scaling_list_8x8[0], pps->scaling_matrix8[0], sizeof(controls->scaling_matrix.scaling_list_8x8[0])); -+ memcpy(controls->scaling_matrix.scaling_list_8x8[1], pps->scaling_matrix8[3], sizeof(controls->scaling_matrix.scaling_list_8x8[1])); -+ -+ if (sps->chroma_format_idc == 3) { -+ memcpy(controls->scaling_matrix.scaling_list_8x8[2], pps->scaling_matrix8[1], sizeof(controls->scaling_matrix.scaling_list_8x8[2])); -+ memcpy(controls->scaling_matrix.scaling_list_8x8[3], pps->scaling_matrix8[4], sizeof(controls->scaling_matrix.scaling_list_8x8[3])); -+ memcpy(controls->scaling_matrix.scaling_list_8x8[4], pps->scaling_matrix8[2], sizeof(controls->scaling_matrix.scaling_list_8x8[4])); -+ memcpy(controls->scaling_matrix.scaling_list_8x8[5], pps->scaling_matrix8[5], sizeof(controls->scaling_matrix.scaling_list_8x8[5])); -+ } -+ -+ controls->decode_params = (struct v4l2_ctrl_h264_decode_params) { -+ .nal_ref_idc = h->nal_ref_idc, -+ .frame_num = h->poc.frame_num, -+ .top_field_order_cnt = h->cur_pic_ptr->field_poc[0] != INT_MAX ? h->cur_pic_ptr->field_poc[0] : 0, -+ .bottom_field_order_cnt = h->cur_pic_ptr->field_poc[1] != INT_MAX ? h->cur_pic_ptr->field_poc[1] : 0, -+ .idr_pic_id = sl->idr_pic_id, -+ .pic_order_cnt_lsb = sl->poc_lsb, -+ .delta_pic_order_cnt_bottom = sl->delta_poc_bottom, -+ .delta_pic_order_cnt0 = sl->delta_poc[0], -+ .delta_pic_order_cnt1 = sl->delta_poc[1], -+ /* Size in bits of dec_ref_pic_marking() syntax element. */ -+ .dec_ref_pic_marking_bit_size = sl->ref_pic_marking_bit_size, -+ /* Size in bits of pic order count syntax. */ -+ .pic_order_cnt_bit_size = sl->pic_order_cnt_bit_size, -+ .slice_group_change_cycle = 0, /* slice group not supported by FFmpeg */ -+ }; -+ -+ if (h->picture_idr) -+ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; -+ if (FIELD_PICTURE(h)) -+ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC; -+ if (h->picture_structure == PICT_BOTTOM_FIELD) -+ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD; -+ if (sl->slice_type_nos == AV_PICTURE_TYPE_P) -+ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_PFRAME; -+ if (sl->slice_type_nos == AV_PICTURE_TYPE_B) -+ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BFRAME; -+ -+ fill_dpb(&controls->decode_params, h); -+ -+ controls->first_slice = 1; -+ controls->num_slices = 0; -+ -+ return ff_v4l2_request_reset_frame(avctx, h->cur_pic_ptr->f); -+} -+ -+static int v4l2_request_h264_queue_decode(AVCodecContext *avctx, int last_slice) -+{ -+ const H264Context *h = avctx->priv_data; -+ V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; -+ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; -+ -+ struct v4l2_ext_control control[] = { -+ { -+ .id = V4L2_CID_STATELESS_H264_SPS, -+ .ptr = &controls->sps, -+ .size = sizeof(controls->sps), -+ }, -+ { -+ .id = V4L2_CID_STATELESS_H264_PPS, -+ .ptr = &controls->pps, -+ .size = sizeof(controls->pps), -+ }, -+ { -+ .id = V4L2_CID_STATELESS_H264_SCALING_MATRIX, -+ .ptr = &controls->scaling_matrix, -+ .size = sizeof(controls->scaling_matrix), -+ }, -+ { -+ .id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, -+ .ptr = &controls->decode_params, -+ .size = sizeof(controls->decode_params), -+ }, -+ { -+ .id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, -+ .ptr = &controls->slice_params, -+ .size = sizeof(controls->slice_params), -+ }, -+ { -+ .id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS, -+ .ptr = &controls->pred_weights, -+ .size = sizeof(controls->pred_weights), -+ }, -+ }; -+ -+ if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) { -+ int count = FF_ARRAY_ELEMS(control) - (controls->pred_weights_required ? 0 : 1); -+ return ff_v4l2_request_decode_slice(avctx, h->cur_pic_ptr->f, control, count, controls->first_slice, last_slice); -+ } -+ -+ return ff_v4l2_request_decode_frame(avctx, h->cur_pic_ptr->f, control, FF_ARRAY_ELEMS(control) - 2); -+} -+ -+static int v4l2_request_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) -+{ -+ const H264Context *h = avctx->priv_data; -+ const PPS *pps = h->ps.pps; -+ const H264SliceContext *sl = &h->slice_ctx[0]; -+ V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; -+ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; -+ int i, ret, count; -+ -+ if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && controls->num_slices) { -+ ret = v4l2_request_h264_queue_decode(avctx, 0); -+ if (ret) -+ return ret; -+ -+ ff_v4l2_request_reset_frame(avctx, h->cur_pic_ptr->f); -+ controls->first_slice = 0; -+ } -+ -+ if (ctx->start_code == V4L2_STATELESS_H264_START_CODE_ANNEX_B) { -+ ret = ff_v4l2_request_append_output_buffer(avctx, h->cur_pic_ptr->f, nalu_slice_start_code, 3); -+ if (ret) -+ return ret; -+ } -+ -+ ret = ff_v4l2_request_append_output_buffer(avctx, h->cur_pic_ptr->f, buffer, size); -+ if (ret) + + return ref; + } +@@ -261,8 +265,7 @@ int ff_v4l2_request_frame_params(AVCodecContext *avctx, + AVHWFramesContext *hwfc = (AVHWFramesContext *)hw_frames_ctx->data; + + hwfc->format = AV_PIX_FMT_DRM_PRIME; +- // TODO: Set sw_format based on capture buffer format +- hwfc->sw_format = AV_PIX_FMT_NONE; ++ hwfc->sw_format = ff_v4l2_request_get_sw_format(&ctx->format); + + if (V4L2_TYPE_IS_MULTIPLANAR(ctx->format.type)) { + hwfc->width = ctx->format.fmt.pix_mp.width; +@@ -413,6 +416,7 @@ int ff_v4l2_request_init(AVCodecContext *avctx, + { + V4L2RequestContext *ctx = v4l2_request_context(avctx); + AVV4L2RequestDeviceContext *hwctx = NULL; ++ int ret; + + // Set initial default values + ctx->av_class = &v4l2_request_context_class; +@@ -428,7 +432,13 @@ int ff_v4l2_request_init(AVCodecContext *avctx, + } + } + +- // TODO: Probe for a capable media and video device ++ // Probe for a capable media and video device for the V4L2 codec pixelformat ++ ret = ff_v4l2_request_probe(avctx, pixelformat, buffersize, control, count); ++ if (ret < 0) { ++ av_log(avctx, AV_LOG_INFO, "No V4L2 video device found for %s\n", ++ av_fourcc2str(pixelformat)); + return ret; ++ } + + // Transfer (or return) ownership of media device file descriptor to hwdevice + if (hwctx) { +diff --git a/libavcodec/v4l2_request_internal.h b/libavcodec/v4l2_request_internal.h +index 554b663ab4..a5c3de22ef 100644 +--- a/libavcodec/v4l2_request_internal.h ++++ b/libavcodec/v4l2_request_internal.h +@@ -39,4 +39,13 @@ static inline V4L2RequestFrameDescriptor *v4l2_request_framedesc(AVFrame *frame) + return (V4L2RequestFrameDescriptor *)frame->data[0]; + } + ++enum AVPixelFormat ff_v4l2_request_get_sw_format(struct v4l2_format *format); + -+ if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) -+ return 0; ++int ff_v4l2_request_set_drm_descriptor(V4L2RequestFrameDescriptor *framedesc, ++ struct v4l2_format *format); + -+ controls->slice_params = (struct v4l2_ctrl_h264_slice_params) { -+ /* Offset in bits to slice_data() from the beginning of this slice. */ -+ .header_bit_size = get_bits_count(&sl->gb), ++int ff_v4l2_request_probe(AVCodecContext *avctx, uint32_t pixelformat, ++ uint32_t buffersize, struct v4l2_ext_control *control, ++ int count); + -+ .first_mb_in_slice = sl->first_mb_addr, + #endif /* AVCODEC_V4L2_REQUEST_INTERNAL_H */ +diff --git a/libavcodec/v4l2_request_probe.c b/libavcodec/v4l2_request_probe.c +new file mode 100644 +index 0000000000..941042ec04 +--- /dev/null ++++ b/libavcodec/v4l2_request_probe.c +@@ -0,0 +1,614 @@ ++/* ++ * This file is part of FFmpeg. ++ * ++ * FFmpeg is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2.1 of the License, or (at your option) any later version. ++ * ++ * FFmpeg is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with FFmpeg; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ + -+ .slice_type = ff_h264_get_slice_type(sl), -+ .colour_plane_id = 0, /* separate colour plane not supported by FFmpeg */ -+ .redundant_pic_cnt = sl->redundant_pic_count, -+ .cabac_init_idc = sl->cabac_init_idc, -+ .slice_qp_delta = sl->qscale - pps->init_qp, -+ .slice_qs_delta = 0, /* not implemented by FFmpeg */ -+ .disable_deblocking_filter_idc = sl->deblocking_filter < 2 ? !sl->deblocking_filter : sl->deblocking_filter, -+ .slice_alpha_c0_offset_div2 = sl->slice_alpha_c0_offset / 2, -+ .slice_beta_offset_div2 = sl->slice_beta_offset / 2, -+ .num_ref_idx_l0_active_minus1 = sl->list_count > 0 ? sl->ref_count[0] - 1 : 0, -+ .num_ref_idx_l1_active_minus1 = sl->list_count > 1 ? sl->ref_count[1] - 1 : 0, -+ }; ++#include "config.h" + -+ if (sl->slice_type == AV_PICTURE_TYPE_B && sl->direct_spatial_mv_pred) -+ controls->slice_params.flags |= V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; ++#include ++#include ++#include ++#include + -+ /* V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH: not implemented by FFmpeg */ ++#include ++#include + -+ controls->pred_weights_required = V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(&controls->pps, &controls->slice_params); -+ if (controls->pred_weights_required) { -+ controls->pred_weights.chroma_log2_weight_denom = sl->pwt.chroma_log2_weight_denom; -+ controls->pred_weights.luma_log2_weight_denom = sl->pwt.luma_log2_weight_denom; ++#include "libavutil/hwcontext_v4l2request.h" ++#include "libavutil/mem.h" ++#include "v4l2_request_internal.h" ++ ++static const struct { ++ uint32_t pixelformat; ++ enum AVPixelFormat sw_format; ++ uint32_t drm_format; ++ uint64_t format_modifier; ++} v4l2_request_capture_pixelformats[] = { ++ { V4L2_PIX_FMT_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_LINEAR }, ++#if defined(V4L2_PIX_FMT_NV12_32L32) ++ { V4L2_PIX_FMT_NV12_32L32, AV_PIX_FMT_NONE, DRM_FORMAT_NV12, DRM_FORMAT_MOD_ALLWINNER_TILED }, ++#endif ++#if defined(V4L2_PIX_FMT_NV15) && defined(DRM_FORMAT_NV15) ++ { V4L2_PIX_FMT_NV15, AV_PIX_FMT_NONE, DRM_FORMAT_NV15, DRM_FORMAT_MOD_LINEAR }, ++#endif ++ { V4L2_PIX_FMT_NV16, AV_PIX_FMT_NV16, DRM_FORMAT_NV16, DRM_FORMAT_MOD_LINEAR }, ++#if defined(V4L2_PIX_FMT_NV20) && defined(DRM_FORMAT_NV20) ++ { V4L2_PIX_FMT_NV20, AV_PIX_FMT_NONE, DRM_FORMAT_NV20, DRM_FORMAT_MOD_LINEAR }, ++#endif ++#if defined(V4L2_PIX_FMT_P010) && defined(DRM_FORMAT_P010) ++ { V4L2_PIX_FMT_P010, AV_PIX_FMT_P010, DRM_FORMAT_P010, DRM_FORMAT_MOD_LINEAR }, ++#endif ++#if defined(V4L2_PIX_FMT_NV12_COL128) && defined(V4L2_PIX_FMT_NV12_10_COL128) ++ { ++ .pixelformat = V4L2_PIX_FMT_NV12_COL128, ++ .sw_format = AV_PIX_FMT_NONE, ++ .drm_format = DRM_FORMAT_NV12, ++ .format_modifier = DRM_FORMAT_MOD_BROADCOM_SAND128, ++ }, ++#if defined(DRM_FORMAT_P030) ++ { ++ .pixelformat = V4L2_PIX_FMT_NV12_10_COL128, ++ .sw_format = AV_PIX_FMT_NONE, ++ .drm_format = DRM_FORMAT_P030, ++ .format_modifier = DRM_FORMAT_MOD_BROADCOM_SAND128, ++ }, ++#endif ++#endif ++#if defined(V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT) ++ { ++ .pixelformat = V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT, ++ .sw_format = AV_PIX_FMT_NONE, ++ .drm_format = DRM_FORMAT_YUV420_10BIT, ++ .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | ++ AFBC_FORMAT_MOD_SPARSE | ++ AFBC_FORMAT_MOD_SPLIT), ++ }, ++#endif ++#if defined(V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT) ++ { ++ .pixelformat = V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT, ++ .sw_format = AV_PIX_FMT_NONE, ++ .drm_format = DRM_FORMAT_YUV420_8BIT, ++ .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | ++ AFBC_FORMAT_MOD_SPARSE | ++ AFBC_FORMAT_MOD_SPLIT), ++ }, ++#endif ++}; ++ ++enum AVPixelFormat ff_v4l2_request_get_sw_format(struct v4l2_format *format) ++{ ++ uint32_t pixelformat = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? ++ format->fmt.pix_mp.pixelformat : ++ format->fmt.pix.pixelformat; ++ ++ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { ++ if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) ++ return v4l2_request_capture_pixelformats[i].sw_format; + } + -+ count = sl->list_count > 0 ? sl->ref_count[0] : 0; -+ for (i = 0; i < count; i++) -+ fill_ref_list(&controls->slice_params.ref_pic_list0[i], &controls->decode_params, &sl->ref_list[0][i]); -+ if (count && controls->pred_weights_required) -+ fill_weight_factors(&controls->pred_weights.weight_factors[0], 0, sl); ++ return AV_PIX_FMT_NONE; ++} + -+ count = sl->list_count > 1 ? sl->ref_count[1] : 0; -+ for (i = 0; i < count; i++) -+ fill_ref_list(&controls->slice_params.ref_pic_list1[i], &controls->decode_params, &sl->ref_list[1][i]); -+ if (count && controls->pred_weights_required) -+ fill_weight_factors(&controls->pred_weights.weight_factors[1], 1, sl); ++int ff_v4l2_request_set_drm_descriptor(V4L2RequestFrameDescriptor *framedesc, ++ struct v4l2_format *format) ++{ ++ AVDRMFrameDescriptor *desc = &framedesc->base; ++ AVDRMLayerDescriptor *layer = &desc->layers[0]; ++ uint32_t pixelformat = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? ++ format->fmt.pix_mp.pixelformat : ++ format->fmt.pix.pixelformat; ++ ++ // Set drm format and format modifier ++ layer->format = 0; ++ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { ++ if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) { ++ layer->format = v4l2_request_capture_pixelformats[i].drm_format; ++ desc->objects[0].format_modifier = ++ v4l2_request_capture_pixelformats[i].format_modifier; ++ break; ++ } ++ } ++ ++ if (!layer->format) ++ return AVERROR(ENOENT); ++ ++ desc->nb_objects = 1; ++ desc->objects[0].fd = framedesc->capture.fd; ++ desc->objects[0].size = framedesc->capture.size; ++ ++ desc->nb_layers = 1; ++ layer->nb_planes = 1; ++ ++ layer->planes[0].object_index = 0; ++ layer->planes[0].offset = 0; ++ layer->planes[0].pitch = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? ++ format->fmt.pix_mp.plane_fmt[0].bytesperline : ++ format->fmt.pix.bytesperline; ++ ++ // AFBC formats only use 1 plane, remaining use 2 planes ++ if ((desc->objects[0].format_modifier >> 56) != DRM_FORMAT_MOD_VENDOR_ARM) { ++ layer->nb_planes = 2; ++ layer->planes[1].object_index = 0; ++ layer->planes[1].offset = layer->planes[0].pitch * ++ (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? ++ format->fmt.pix_mp.height : ++ format->fmt.pix.height); ++ layer->planes[1].pitch = layer->planes[0].pitch; ++ } ++ ++#if defined(V4L2_PIX_FMT_NV12_COL128) && defined(V4L2_PIX_FMT_NV12_10_COL128) ++ // Raspberry Pi formats need special handling ++ if (pixelformat == V4L2_PIX_FMT_NV12_COL128 || ++ pixelformat == V4L2_PIX_FMT_NV12_10_COL128) { ++ desc->objects[0].format_modifier = ++ DRM_FORMAT_MOD_BROADCOM_SAND128_COL_HEIGHT(layer->planes[0].pitch); ++ layer->planes[1].offset = 128 * ++ (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? ++ format->fmt.pix_mp.height : ++ format->fmt.pix.height); ++ layer->planes[0].pitch = (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? ++ format->fmt.pix_mp.width : ++ format->fmt.pix.width); ++ if (pixelformat == V4L2_PIX_FMT_NV12_10_COL128) ++ layer->planes[0].pitch *= 2; ++ layer->planes[1].pitch = layer->planes[0].pitch; ++ } ++#endif + -+ controls->num_slices++; + return 0; +} + -+static int v4l2_request_h264_end_frame(AVCodecContext *avctx) ++static int v4l2_request_set_format(AVCodecContext *avctx, ++ enum v4l2_buf_type type, ++ uint32_t pixelformat, ++ uint32_t buffersize) +{ -+ return v4l2_request_h264_queue_decode(avctx, 1); -+} -+ -+static int v4l2_request_h264_set_controls(AVCodecContext *avctx) -+{ -+ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; -+ -+ struct v4l2_ext_control control[] = { -+ { .id = V4L2_CID_STATELESS_H264_DECODE_MODE, }, -+ { .id = V4L2_CID_STATELESS_H264_START_CODE, }, ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct v4l2_format format = { ++ .type = type, + }; + -+ ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, V4L2_CID_STATELESS_H264_DECODE_MODE); -+ if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && -+ ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED) { -+ av_log(avctx, AV_LOG_ERROR, "%s: unsupported decode mode, %d\n", __func__, ctx->decode_mode); -+ return AVERROR(EINVAL); ++ if (V4L2_TYPE_IS_MULTIPLANAR(type)) { ++ format.fmt.pix_mp.width = avctx->coded_width; ++ format.fmt.pix_mp.height = avctx->coded_height; ++ format.fmt.pix_mp.pixelformat = pixelformat; ++ format.fmt.pix_mp.plane_fmt[0].sizeimage = buffersize; ++ format.fmt.pix_mp.num_planes = 1; ++ } else { ++ format.fmt.pix.width = avctx->coded_width; ++ format.fmt.pix.height = avctx->coded_height; ++ format.fmt.pix.pixelformat = pixelformat; ++ format.fmt.pix.sizeimage = buffersize; + } + -+ ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, V4L2_CID_STATELESS_H264_START_CODE); -+ if (ctx->start_code != V4L2_STATELESS_H264_START_CODE_NONE && -+ ctx->start_code != V4L2_STATELESS_H264_START_CODE_ANNEX_B) { -+ av_log(avctx, AV_LOG_ERROR, "%s: unsupported start code, %d\n", __func__, ctx->start_code); -+ return AVERROR(EINVAL); -+ } ++ if (ioctl(ctx->video_fd, VIDIOC_S_FMT, &format) < 0) ++ return AVERROR(errno); + -+ control[0].value = ctx->decode_mode; -+ control[1].value = ctx->start_code; -+ -+ return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control)); ++ return 0; +} + -+static int v4l2_request_h264_init(AVCodecContext *avctx) ++static int v4l2_request_select_capture_format(AVCodecContext *avctx) +{ -+ const H264Context *h = avctx->priv_data; -+ struct v4l2_ctrl_h264_sps sps; ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ enum v4l2_buf_type type = ctx->format.type; ++ struct v4l2_format format = { ++ .type = type, ++ }; ++ struct v4l2_fmtdesc fmtdesc = { ++ .index = 0, ++ .type = type, ++ }; ++ uint32_t pixelformat; ++ ++ // Get the driver preferred (or default) format ++ if (ioctl(ctx->video_fd, VIDIOC_G_FMT, &format) < 0) ++ return AVERROR(errno); ++ ++ pixelformat = V4L2_TYPE_IS_MULTIPLANAR(type) ? ++ format.fmt.pix_mp.pixelformat : ++ format.fmt.pix.pixelformat; ++ ++ // Use the driver preferred format when it is supported ++ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { ++ if (pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) ++ return v4l2_request_set_format(avctx, type, pixelformat, 0); ++ } ++ ++ // Otherwise, use first format that is supported ++ while (ioctl(ctx->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) { ++ for (int i = 0; i < FF_ARRAY_ELEMS(v4l2_request_capture_pixelformats); i++) { ++ if (fmtdesc.pixelformat == v4l2_request_capture_pixelformats[i].pixelformat) ++ return v4l2_request_set_format(avctx, type, fmtdesc.pixelformat, 0); ++ } ++ ++ fmtdesc.index++; ++ } ++ ++ return AVERROR(errno); ++} ++ ++static int v4l2_request_try_framesize(AVCodecContext *avctx, ++ uint32_t pixelformat) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct v4l2_frmsizeenum frmsize = { ++ .index = 0, ++ .pixel_format = pixelformat, ++ }; ++ ++ // Enumerate and check if frame size is supported ++ while (ioctl(ctx->video_fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) >= 0) { ++ if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE && ++ frmsize.discrete.width == avctx->coded_width && ++ frmsize.discrete.height == avctx->coded_height) { ++ return 0; ++ } else if ((frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE || ++ frmsize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) && ++ avctx->coded_width <= frmsize.stepwise.max_width && ++ avctx->coded_height <= frmsize.stepwise.max_height && ++ avctx->coded_width % frmsize.stepwise.step_width == 0 && ++ avctx->coded_height % frmsize.stepwise.step_height == 0) { ++ return 0; ++ } ++ ++ frmsize.index++; ++ } ++ ++ return AVERROR(errno); ++} ++ ++static int v4l2_request_try_format(AVCodecContext *avctx, ++ enum v4l2_buf_type type, ++ uint32_t pixelformat) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct v4l2_fmtdesc fmtdesc = { ++ .index = 0, ++ .type = type, ++ }; ++ ++ // Enumerate and check if format is supported ++ while (ioctl(ctx->video_fd, VIDIOC_ENUM_FMT, &fmtdesc) >= 0) { ++ if (fmtdesc.pixelformat == pixelformat) ++ return 0; ++ ++ fmtdesc.index++; ++ } ++ ++ return AVERROR(errno); ++} ++ ++static int v4l2_request_probe_video_device(const char *path, ++ AVCodecContext *avctx, ++ uint32_t pixelformat, ++ uint32_t buffersize, ++ struct v4l2_ext_control *control, ++ int count) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct v4l2_capability capability; ++ struct v4l2_create_buffers buffers = { ++ .count = 0, ++ .memory = V4L2_MEMORY_MMAP, ++ }; ++ unsigned int capabilities; + int ret; + -+ struct v4l2_ext_control control[] = { -+ { -+ .id = V4L2_CID_STATELESS_H264_SPS, -+ .ptr = &sps, -+ .size = sizeof(sps), -+ }, ++ /* ++ * Open video device in non-blocking mode to support decoding using ++ * multiple queued requests, required for e.g. multi stage decoding. ++ */ ++ ctx->video_fd = open(path, O_RDWR | O_NONBLOCK); ++ if (ctx->video_fd < 0) { ++ av_log(avctx, AV_LOG_ERROR, "Failed to open video device %s: %s (%d)\n", ++ path, strerror(errno), errno); ++ return AVERROR(errno); ++ } ++ ++ // Query capabilities of the video device ++ if (ioctl(ctx->video_fd, VIDIOC_QUERYCAP, &capability) < 0) { ++ av_log(avctx, AV_LOG_ERROR, "Failed to query capabilities of %s: %s (%d)\n", ++ path, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ // Use device capabilities when needed ++ if (capability.capabilities & V4L2_CAP_DEVICE_CAPS) ++ capabilities = capability.device_caps; ++ else ++ capabilities = capability.capabilities; ++ ++ // Ensure streaming is supported on the video device ++ if ((capabilities & V4L2_CAP_STREAMING) != V4L2_CAP_STREAMING) { ++ av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing streaming capability\n", path); ++ ret = AVERROR(EINVAL); ++ goto fail; ++ } ++ ++ // Ensure multi- or single-planar API can be used ++ if ((capabilities & V4L2_CAP_VIDEO_M2M_MPLANE) == V4L2_CAP_VIDEO_M2M_MPLANE) { ++ ctx->output_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; ++ ctx->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; ++ } else if ((capabilities & V4L2_CAP_VIDEO_M2M) == V4L2_CAP_VIDEO_M2M) { ++ ctx->output_type = V4L2_BUF_TYPE_VIDEO_OUTPUT; ++ ctx->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; ++ } else { ++ av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing mem2mem capability\n", path); ++ ret = AVERROR(EINVAL); ++ goto fail; ++ } ++ ++ // Query output buffer capabilities ++ buffers.format.type = ctx->output_type; ++ if (ioctl(ctx->video_fd, VIDIOC_CREATE_BUFS, &buffers) < 0) { ++ av_log(avctx, AV_LOG_ERROR, ++ "Failed to query output buffer capabilities of %s: %s (%d)\n", ++ path, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ // Ensure requests can be used ++ if ((buffers.capabilities & V4L2_BUF_CAP_SUPPORTS_REQUESTS) != ++ V4L2_BUF_CAP_SUPPORTS_REQUESTS) { ++ av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing support for requests\n", path); ++ ret = AVERROR(EINVAL); ++ goto fail; ++ } ++ ++ // Ensure the codec pixelformat can be used ++ ret = v4l2_request_try_format(avctx, ctx->output_type, pixelformat); ++ if (ret < 0) { ++ av_log(ctx, AV_LOG_VERBOSE, "Device %s is missing support for pixelformat %s\n", ++ path, av_fourcc2str(pixelformat)); ++ goto fail; ++ } ++ ++ // Ensure frame size is supported, when driver support ENUM_FRAMESIZES ++ ret = v4l2_request_try_framesize(avctx, pixelformat); ++ if (ret < 0 && ret != AVERROR(ENOTTY)) { ++ av_log(ctx, AV_LOG_VERBOSE, ++ "Device %s is missing support for frame size %dx%d of pixelformat %s\n", ++ path, avctx->coded_width, avctx->coded_height, av_fourcc2str(pixelformat)); ++ goto fail; ++ } ++ ++ // Set the codec pixelformat and output buffersize to be used ++ ret = v4l2_request_set_format(avctx, ctx->output_type, pixelformat, buffersize); ++ if (ret < 0) { ++ av_log(avctx, AV_LOG_ERROR, ++ "Failed to set output pixelformat %s of %s: %s (%d)\n", ++ av_fourcc2str(pixelformat), path, strerror(errno), errno); ++ goto fail; ++ } ++ ++ /* ++ * Set any codec specific controls that can help assist the driver ++ * make a decision on what capture buffer format can be used. ++ */ ++ ret = ff_v4l2_request_set_controls(avctx, control, count); ++ if (ret < 0) ++ goto fail; ++ ++ // Select a capture buffer format known to the hwaccel ++ ret = v4l2_request_select_capture_format(avctx); ++ if (ret < 0) { ++ av_log(avctx, AV_LOG_VERBOSE, ++ "Failed to select a capture format for %s of %s: %s (%d)\n", ++ av_fourcc2str(pixelformat), path, strerror(errno), errno); ++ goto fail; ++ } ++ ++ // Check codec specific controls, e.g. profile and level ++ if (ctx->post_probe) { ++ ret = ctx->post_probe(avctx); ++ if (ret < 0) ++ goto fail; ++ } ++ ++ // All tests passed, video device should be capable ++ return 0; ++ ++fail: ++ if (ctx->video_fd >= 0) { ++ close(ctx->video_fd); ++ ctx->video_fd = -1; ++ } ++ return ret; ++} ++ ++static int v4l2_request_probe_video_devices(struct udev *udev, ++ AVCodecContext *avctx, ++ uint32_t pixelformat, ++ uint32_t buffersize, ++ struct v4l2_ext_control *control, ++ int count) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct media_device_info device_info; ++ struct media_v2_topology topology = {0}; ++ struct media_v2_interface *interfaces; ++ struct udev_device *device; ++ const char *path; ++ dev_t devnum; ++ int ret; ++ ++ if (ioctl(ctx->media_fd, MEDIA_IOC_DEVICE_INFO, &device_info) < 0) ++ return AVERROR(errno); ++ ++ if (ioctl(ctx->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology) < 0) { ++ av_log(avctx, AV_LOG_ERROR, "Failed to get media topology: %s (%d)\n", ++ strerror(errno), errno); ++ return AVERROR(errno); ++ } ++ ++ if (!topology.num_interfaces) ++ return AVERROR(ENOENT); ++ ++ interfaces = av_calloc(topology.num_interfaces, sizeof(struct media_v2_interface)); ++ if (!interfaces) ++ return AVERROR(ENOMEM); ++ ++ topology.ptr_interfaces = (__u64)(uintptr_t)interfaces; ++ if (ioctl(ctx->media_fd, MEDIA_IOC_G_TOPOLOGY, &topology) < 0) { ++ av_log(avctx, AV_LOG_ERROR, "Failed to get media topology: %s (%d)\n", ++ strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ ret = AVERROR(ENOENT); ++ for (int i = 0; i < topology.num_interfaces; i++) { ++ if (interfaces[i].intf_type != MEDIA_INTF_T_V4L_VIDEO) ++ continue; ++ ++ devnum = makedev(interfaces[i].devnode.major, interfaces[i].devnode.minor); ++ device = udev_device_new_from_devnum(udev, 'c', devnum); ++ if (!device) ++ continue; ++ ++ path = udev_device_get_devnode(device); ++ if (path) ++ ret = v4l2_request_probe_video_device(path, avctx, pixelformat, ++ buffersize, control, count); ++ udev_device_unref(device); ++ ++ // Stop when we have found a capable video device ++ if (!ret) { ++ av_log(avctx, AV_LOG_INFO, ++ "Using V4L2 media driver %s (%u.%u.%u) for %s\n", ++ device_info.driver, ++ device_info.driver_version >> 16, ++ (device_info.driver_version >> 8) & 0xff, ++ device_info.driver_version & 0xff, ++ av_fourcc2str(pixelformat)); ++ break; ++ } ++ } ++ ++fail: ++ av_free(interfaces); ++ return ret; ++} ++ ++static int v4l2_request_probe_media_device(struct udev_device *device, ++ AVCodecContext *avctx, ++ uint32_t pixelformat, ++ uint32_t buffersize, ++ struct v4l2_ext_control *control, ++ int count) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ const char *path; ++ int ret; ++ ++ path = udev_device_get_devnode(device); ++ if (!path) ++ return AVERROR(ENODEV); ++ ++ // Open enumerated media device ++ ctx->media_fd = open(path, O_RDWR); ++ if (ctx->media_fd < 0) { ++ av_log(avctx, AV_LOG_ERROR, "Failed to open media device %s: %s (%d)\n", ++ path, strerror(errno), errno); ++ return AVERROR(errno); ++ } ++ ++ // Probe video devices of current media device ++ ret = v4l2_request_probe_video_devices(udev_device_get_udev(device), ++ avctx, pixelformat, ++ buffersize, control, count); ++ ++ // Cleanup when no capable video device was found ++ if (ret < 0) { ++ close(ctx->media_fd); ++ ctx->media_fd = -1; ++ } ++ ++ return ret; ++} ++ ++static int v4l2_request_probe_media_devices(struct udev *udev, ++ AVCodecContext *avctx, ++ uint32_t pixelformat, ++ uint32_t buffersize, ++ struct v4l2_ext_control *control, ++ int count) ++{ ++ struct udev_enumerate *enumerate; ++ struct udev_list_entry *devices; ++ struct udev_list_entry *entry; ++ struct udev_device *device; ++ int ret; ++ ++ enumerate = udev_enumerate_new(udev); ++ if (!enumerate) ++ return AVERROR(ENOMEM); ++ ++ udev_enumerate_add_match_subsystem(enumerate, "media"); ++ udev_enumerate_scan_devices(enumerate); ++ devices = udev_enumerate_get_list_entry(enumerate); ++ ++ ret = AVERROR(ENOENT); ++ udev_list_entry_foreach(entry, devices) { ++ const char *path = udev_list_entry_get_name(entry); ++ if (!path) ++ continue; ++ ++ device = udev_device_new_from_syspath(udev, path); ++ if (!device) ++ continue; ++ ++ // Probe media device for a capable video device ++ ret = v4l2_request_probe_media_device(device, avctx, pixelformat, ++ buffersize, control, count); ++ udev_device_unref(device); ++ ++ // Stop when we have found a capable media and video device ++ if (!ret) ++ break; ++ } ++ ++ udev_enumerate_unref(enumerate); ++ return ret; ++} ++ ++int ff_v4l2_request_probe(AVCodecContext *avctx, ++ uint32_t pixelformat, uint32_t buffersize, ++ struct v4l2_ext_control *control, int count) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct udev *udev; ++ int ret; ++ ++ udev = udev_new(); ++ if (!udev) ++ return AVERROR(ENOMEM); ++ ++ if (ctx->media_fd >= 0) { ++ // Probe video devices of current media device ++ ret = v4l2_request_probe_video_devices(udev, avctx, pixelformat, ++ buffersize, control, count); ++ } else { ++ // Probe all media devices (auto-detect) ++ ret = v4l2_request_probe_media_devices(udev, avctx, pixelformat, ++ buffersize, control, count); ++ } ++ ++ udev_unref(udev); ++ return ret; ++} + +From 917168b391fa76cff0b7b74050bd90fa60e07b8c Mon Sep 17 00:00:00 2001 +From: Jonas Karlman +Date: Tue, 6 Aug 2024 09:06:03 +0000 +Subject: [PATCH 04/11] avcodec/v4l2request: Add common decode support for + hwaccels + +Add common support for decoding using the V4L2 Request API. + +Basic flow for decoding follow the kernel Memory-to-memory Stateless +Video Decoder Interface > Decoding [1]. + +A codec hwaccel typically handle decoding as follow: + +In start_frame next OUTPUT buffer and its related request object is +picked from a circular queue and any codec specific CONTROLs is prepared. + +In decode_slice the slice bitstream data is appended to the OUTPUT +buffer. + +In end_frame a CAPTURE buffer tied to the AVFrame is queued, it will be +used as the decoding target by the driver / hw decoder. The prepared +codec specific CONTROLs get queued as part of the request object. +Finally the request object is submitted for decoding. + +For slice based hw decoders only the request for the final slice of the +frame is submitted in end_frame, remaining is submitted in decode_slice. + +[1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#decoding + +Co-developed-by: Jernej Skrabec +Signed-off-by: Jernej Skrabec +Co-developed-by: Alex Bee +Signed-off-by: Alex Bee +Signed-off-by: Jonas Karlman +--- + configure | 3 +- + libavcodec/Makefile | 2 +- + libavcodec/hwconfig.h | 2 + + libavcodec/v4l2_request.c | 3 +- + libavcodec/v4l2_request.h | 23 ++ + libavcodec/v4l2_request_decode.c | 459 +++++++++++++++++++++++++++++++ + 6 files changed, 489 insertions(+), 3 deletions(-) + create mode 100644 libavcodec/v4l2_request_decode.c + +diff --git a/configure b/configure +index 98a342d109..b74da32e8b 100755 +--- a/configure ++++ b/configure +@@ -3149,7 +3149,7 @@ dxva2_deps="dxva2api_h DXVA2_ConfigPictureDecode ole32 user32" + ffnvcodec_deps_any="libdl LoadLibrary" + mediacodec_deps="android mediandk" + nvdec_deps="ffnvcodec" +-v4l2_request_deps="linux_media_h v4l2_timeval_to_ns libdrm libudev" ++v4l2_request_deps="linux_media_h v4l2_timeval_to_ns v4l2_m2m_hold_capture_buf libdrm libudev" + vaapi_x11_deps="xlib_x11" + videotoolbox_hwaccel_deps="videotoolbox pthreads" + videotoolbox_hwaccel_extralibs="-framework QuartzCore" +@@ -7202,6 +7202,7 @@ if enabled v4l2_m2m; then + fi + + if enabled v4l2_request; then ++ check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns + check_pkg_config libudev libudev libudev.h udev_new + fi +diff --git a/libavcodec/Makefile b/libavcodec/Makefile +index ebd2dc54b9..6f2229b1b6 100644 +--- a/libavcodec/Makefile ++++ b/libavcodec/Makefile +@@ -176,7 +176,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o + OBJS-$(CONFIG_VP56DSP) += vp56dsp.o + OBJS-$(CONFIG_VP8DSP) += vp8dsp.o + OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o +-OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o ++OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o v4l2_request_decode.o + OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o + OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o + +diff --git a/libavcodec/hwconfig.h b/libavcodec/hwconfig.h +index ee29ca631d..159064a1f1 100644 +--- a/libavcodec/hwconfig.h ++++ b/libavcodec/hwconfig.h +@@ -79,6 +79,8 @@ void ff_hwaccel_uninit(AVCodecContext *avctx); + HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD, NONE, ff_ ## codec ## _d3d11va_hwaccel) + #define HWACCEL_D3D12VA(codec) \ + HW_CONFIG_HWACCEL(1, 1, 0, D3D12, D3D12VA, ff_ ## codec ## _d3d12va_hwaccel) ++#define HWACCEL_V4L2REQUEST(codec) \ ++ HW_CONFIG_HWACCEL(1, 0, 0, DRM_PRIME, V4L2REQUEST, ff_ ## codec ## _v4l2request_hwaccel) + + #define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \ + &(const AVCodecHWConfigInternal) { \ +diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c +index 6f44c5d826..aae719ead6 100644 +--- a/libavcodec/v4l2_request.c ++++ b/libavcodec/v4l2_request.c +@@ -303,7 +303,8 @@ int ff_v4l2_request_uninit(AVCodecContext *avctx) + V4L2RequestContext *ctx = v4l2_request_context(avctx); + + if (ctx->video_fd >= 0) { +- // TODO: Flush and wait on all pending requests ++ // Flush and wait on all pending requests ++ ff_v4l2_request_flush(avctx); + + // Stop output queue + if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) { +diff --git a/libavcodec/v4l2_request.h b/libavcodec/v4l2_request.h +index 621caaf28c..62248feb48 100644 +--- a/libavcodec/v4l2_request.h ++++ b/libavcodec/v4l2_request.h +@@ -81,4 +81,27 @@ int ff_v4l2_request_init(AVCodecContext *avctx, + uint32_t pixelformat, uint32_t buffersize, + struct v4l2_ext_control *control, int count); + ++uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame); ++ ++int ff_v4l2_request_append_output(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ const uint8_t *data, uint32_t size); ++ ++int ff_v4l2_request_decode_slice(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ struct v4l2_ext_control *control, int count, ++ bool first_slice, bool last_slice); ++ ++int ff_v4l2_request_decode_frame(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ struct v4l2_ext_control *control, int count); ++ ++int ff_v4l2_request_reset_picture(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic); ++ ++int ff_v4l2_request_start_frame(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, AVFrame *frame); ++ ++void ff_v4l2_request_flush(AVCodecContext *avctx); ++ + #endif /* AVCODEC_V4L2_REQUEST_H */ +diff --git a/libavcodec/v4l2_request_decode.c b/libavcodec/v4l2_request_decode.c +new file mode 100644 +index 0000000000..8e7e241e36 +--- /dev/null ++++ b/libavcodec/v4l2_request_decode.c +@@ -0,0 +1,459 @@ ++/* ++ * This file is part of FFmpeg. ++ * ++ * FFmpeg is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2.1 of the License, or (at your option) any later version. ++ * ++ * FFmpeg is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with FFmpeg; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#include "config.h" ++ ++#include ++#include ++ ++#include "decode.h" ++#include "v4l2_request_internal.h" ++ ++#define INPUT_BUFFER_PADDING_SIZE (AV_INPUT_BUFFER_PADDING_SIZE * 4) ++ ++uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame) ++{ ++ V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame); ++ ++ /* ++ * The capture buffer index is used as a base for V4L2 frame reference. ++ * This works because frames are decoded into a capture buffer that is ++ * closely tied to an AVFrame. ++ */ ++ return desc ? v4l2_timeval_to_ns(&desc->capture.buffer.timestamp) : 0; ++} ++ ++static int v4l2_request_queue_buffer(V4L2RequestContext *ctx, int request_fd, ++ V4L2RequestBuffer *buf, uint32_t flags) ++{ ++ struct v4l2_plane planes[1] = {}; ++ struct v4l2_buffer buffer = { ++ .index = buf->index, ++ .type = buf->buffer.type, ++ .memory = buf->buffer.memory, ++ .timestamp = buf->buffer.timestamp, ++ .bytesused = buf->used, ++ .request_fd = request_fd, ++ .flags = ((request_fd >= 0) ? V4L2_BUF_FLAG_REQUEST_FD : 0) | flags, + }; + -+ fill_sps(&sps, h); ++ if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) { ++ planes[0].bytesused = buf->used; ++ buffer.bytesused = 0; ++ buffer.length = 1; ++ buffer.m.planes = planes; ++ } + -+ ret = ff_v4l2_request_init(avctx, V4L2_PIX_FMT_H264_SLICE, 4 * 1024 * 1024, control, FF_ARRAY_ELEMS(control)); ++ // Queue the buffer ++ if (ioctl(ctx->video_fd, VIDIOC_QBUF, &buffer) < 0) ++ return AVERROR(errno); ++ ++ // Mark the buffer as queued ++ if (V4L2_TYPE_IS_OUTPUT(buffer.type)) ++ atomic_fetch_or(&ctx->queued_output, 1 << buffer.index); ++ else ++ atomic_fetch_or(&ctx->queued_capture, 1 << buffer.index); ++ ++ return 0; ++} ++ ++static int v4l2_request_dequeue_buffer(V4L2RequestContext *ctx, ++ enum v4l2_buf_type type) ++{ ++ struct v4l2_plane planes[1] = {}; ++ struct v4l2_buffer buffer = { ++ .type = type, ++ .memory = V4L2_MEMORY_MMAP, ++ }; ++ ++ if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) { ++ buffer.length = 1; ++ buffer.m.planes = planes; ++ } ++ ++ // Dequeue next completed buffer ++ if (ioctl(ctx->video_fd, VIDIOC_DQBUF, &buffer) < 0) ++ return AVERROR(errno); ++ ++ // Mark the buffer as dequeued ++ if (V4L2_TYPE_IS_OUTPUT(buffer.type)) ++ atomic_fetch_and(&ctx->queued_output, ~(1 << buffer.index)); ++ else ++ atomic_fetch_and(&ctx->queued_capture, ~(1 << buffer.index)); ++ ++ return 0; ++} ++ ++static inline int v4l2_request_dequeue_completed_buffers(V4L2RequestContext *ctx, ++ enum v4l2_buf_type type) ++{ ++ int ret; ++ ++ do { ++ ret = v4l2_request_dequeue_buffer(ctx, type); ++ } while (!ret); ++ ++ return ret; ++} ++ ++static int v4l2_request_wait_on_capture(V4L2RequestContext *ctx, ++ V4L2RequestBuffer *capture) ++{ ++ struct pollfd pollfd = { ++ .fd = ctx->video_fd, ++ .events = POLLIN, ++ }; ++ ++ ff_mutex_lock(&ctx->mutex); ++ ++ // Dequeue all completed capture buffers ++ if (atomic_load(&ctx->queued_capture)) ++ v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type); ++ ++ // Wait on the specific capture buffer, when needed ++ while (atomic_load(&ctx->queued_capture) & (1 << capture->index)) { ++ int ret = poll(&pollfd, 1, 2000); ++ if (ret <= 0) ++ goto fail; ++ ++ ret = v4l2_request_dequeue_buffer(ctx, ctx->format.type); ++ if (ret < 0 && ret != AVERROR(EAGAIN)) ++ goto fail; ++ } ++ ++ ff_mutex_unlock(&ctx->mutex); ++ return 0; ++ ++fail: ++ ff_mutex_unlock(&ctx->mutex); ++ av_log(ctx, AV_LOG_ERROR, "Failed waiting on capture buffer %d\n", ++ capture->index); ++ return AVERROR(EINVAL); ++} ++ ++static V4L2RequestBuffer *v4l2_request_next_output(V4L2RequestContext *ctx) ++{ ++ int index; ++ V4L2RequestBuffer *output; ++ struct pollfd pollfd = { ++ .fd = ctx->video_fd, ++ .events = POLLOUT, ++ }; ++ ++ ff_mutex_lock(&ctx->mutex); ++ ++ // Use next output buffer in the circular queue ++ index = atomic_load(&ctx->next_output); ++ output = &ctx->output[index]; ++ atomic_store(&ctx->next_output, (index + 1) % FF_ARRAY_ELEMS(ctx->output)); ++ ++ // Dequeue all completed output buffers ++ if (atomic_load(&ctx->queued_output)) ++ v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type); ++ ++ // Wait on the specific output buffer, when needed ++ while (atomic_load(&ctx->queued_output) & (1 << output->index)) { ++ int ret = poll(&pollfd, 1, 2000); ++ if (ret <= 0) ++ goto fail; ++ ++ ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type); ++ if (ret < 0 && ret != AVERROR(EAGAIN)) ++ goto fail; ++ } ++ ++ ff_mutex_unlock(&ctx->mutex); ++ ++ // Reset used state ++ output->used = 0; ++ ++ return output; ++ ++fail: ++ ff_mutex_unlock(&ctx->mutex); ++ av_log(ctx, AV_LOG_ERROR, "Failed waiting on output buffer %d\n", ++ output->index); ++ return NULL; ++} ++ ++static int v4l2_request_wait_on_request(V4L2RequestContext *ctx, ++ V4L2RequestBuffer *output) ++{ ++ struct pollfd pollfd = { ++ .fd = output->fd, ++ .events = POLLPRI, ++ }; ++ ++ // Wait on the specific request to complete, when needed ++ while (atomic_load(&ctx->queued_request) & (1 << output->index)) { ++ int ret = poll(&pollfd, 1, 2000); ++ if (ret <= 0) ++ break; ++ ++ // Mark request as dequeued ++ if (pollfd.revents & (POLLPRI | POLLERR)) { ++ atomic_fetch_and(&ctx->queued_request, ~(1 << output->index)); ++ break; ++ } ++ } ++ ++ // Reinit the request object ++ if (ioctl(output->fd, MEDIA_REQUEST_IOC_REINIT, NULL) < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to reinit request object %d: %s (%d)\n", ++ output->fd, strerror(errno), errno); ++ return AVERROR(errno); ++ } ++ ++ // Ensure request is marked as dequeued ++ atomic_fetch_and(&ctx->queued_request, ~(1 << output->index)); ++ ++ return 0; ++} ++ ++int ff_v4l2_request_append_output(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ const uint8_t *data, uint32_t size) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ ++ // Append data to output buffer and ensure there is enough space for padding ++ if (pic->output->used + size + INPUT_BUFFER_PADDING_SIZE <= pic->output->size) { ++ memcpy(pic->output->addr + pic->output->used, data, size); ++ pic->output->used += size; ++ return 0; ++ } else { ++ av_log(ctx, AV_LOG_ERROR, ++ "Failed to append %u bytes data to output buffer %d (%u of %u used)\n", ++ size, pic->output->index, pic->output->used, pic->output->size); ++ return AVERROR(ENOMEM); ++ } ++} ++ ++static int v4l2_request_queue_decode(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ struct v4l2_ext_control *control, int count, ++ bool first_slice, bool last_slice) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ uint32_t flags; ++ int ret; ++ ++ if (first_slice) { ++ /* ++ * Wait on dequeue of the target capture buffer, when needed. Otherwise ++ * V4L2 decoder may use a different capture buffer than hwaccel expects. ++ * ++ * Normally decoding has already completed when a capture buffer is ++ * reused so this is more or less a no-op, however in some situations ++ * FFmpeg may reuse an AVFrame early, i.e. when no output frame was ++ * produced prior time, and a syncronization is necessary. ++ */ ++ ret = v4l2_request_wait_on_capture(ctx, pic->capture); ++ if (ret < 0) ++ return ret; ++ } ++ ++ ff_mutex_lock(&ctx->mutex); ++ ++ /* ++ * The output buffer tied to prior use of current request object can ++ * independently be dequeued before the full decode request has been ++ * completed. This may happen when a decoder use multi stage decoding, ++ * e.g. rpivid. In such case we can start reusing the output buffer, ++ * however we must wait on the prior request to fully complete before we ++ * can reuse the request object, and a syncronization is necessary. ++ */ ++ ret = v4l2_request_wait_on_request(ctx, pic->output); ++ if (ret < 0) ++ goto fail; ++ ++ /* ++ * Dequeue any completed output buffers, this is strictly not necessary, ++ * however if a syncronization was necessary for the capture and/or request ++ * there is more than likely one or more output buffers that can be dequeued. ++ */ ++ if (atomic_load(&ctx->queued_output)) ++ v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type); ++ ++ // Set codec controls for current request ++ ret = ff_v4l2_request_set_request_controls(ctx, pic->output->fd, control, count); ++ if (ret < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s) for request %d: %s (%d)\n", ++ count, pic->output->fd, strerror(errno), errno); ++ goto fail; ++ } ++ ++ // Ensure there is zero padding at the end of bitstream data ++ memset(pic->output->addr + pic->output->used, 0, INPUT_BUFFER_PADDING_SIZE); ++ ++ // Use timestamp of the capture buffer for V4L2 frame reference ++ pic->output->buffer.timestamp = pic->capture->buffer.timestamp; ++ ++ /* ++ * Queue the output buffer of current request. The capture buffer may be ++ * hold by the V4L2 decoder unless this is the last slice of a frame. ++ */ ++ flags = last_slice ? 0 : V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF; ++ ret = v4l2_request_queue_buffer(ctx, pic->output->fd, pic->output, flags); ++ if (ret < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to queue output buffer %d for request %d: %s (%d)\n", ++ pic->output->index, pic->output->fd, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ if (first_slice) { ++ /* ++ * Queue the target capture buffer, hwaccel expect and depend on that ++ * this specific capture buffer will be used as decode target for ++ * current request, otherwise frames may be output in wrong order or ++ * wrong capture buffer could get used as a reference frame. ++ */ ++ ret = v4l2_request_queue_buffer(ctx, -1, pic->capture, 0); ++ if (ret < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to queue capture buffer %d for request %d: %s (%d)\n", ++ pic->capture->index, pic->output->fd, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ } ++ ++ // Queue current request ++ ret = ioctl(pic->output->fd, MEDIA_REQUEST_IOC_QUEUE, NULL); ++ if (ret < 0) { ++ av_log(ctx, AV_LOG_ERROR, "Failed to queue request object %d: %s (%d)\n", ++ pic->output->fd, strerror(errno), errno); ++ ret = AVERROR(errno); ++ goto fail; ++ } ++ ++ // Mark current request as queued ++ atomic_fetch_or(&ctx->queued_request, 1 << pic->output->index); ++ ++ ret = 0; ++fail: ++ ff_mutex_unlock(&ctx->mutex); ++ return ret; ++} ++ ++int ff_v4l2_request_decode_slice(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ struct v4l2_ext_control *control, int count, ++ bool first_slice, bool last_slice) ++{ ++ /* ++ * Fallback to queue each slice as a full frame when holding capture ++ * buffers is not supported by the driver. ++ */ ++ if ((pic->output->capabilities & V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) != ++ V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) ++ return v4l2_request_queue_decode(avctx, pic, control, count, true, true); ++ ++ return v4l2_request_queue_decode(avctx, pic, control, count, ++ first_slice, last_slice); ++} ++ ++int ff_v4l2_request_decode_frame(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ struct v4l2_ext_control *control, int count) ++{ ++ return v4l2_request_queue_decode(avctx, pic, control, count, true, true); ++} ++ ++static int v4l2_request_post_process(void *logctx, AVFrame *frame) ++{ ++ V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame); ++ FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data; ++ V4L2RequestContext *ctx = fdd->hwaccel_priv; ++ ++ // Wait on capture buffer before returning the frame to application ++ return v4l2_request_wait_on_capture(ctx, &desc->capture); ++} ++ ++int ff_v4l2_request_reset_picture(AVCodecContext *avctx, V4L2RequestPictureContext *pic) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ ++ // Get and wait on next output buffer from circular queue ++ pic->output = v4l2_request_next_output(ctx); ++ if (!pic->output) ++ return AVERROR(EINVAL); ++ ++ return 0; ++} ++ ++int ff_v4l2_request_start_frame(AVCodecContext *avctx, ++ V4L2RequestPictureContext *pic, ++ AVFrame *frame) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame); ++ FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data; ++ int ret; ++ ++ // Get next output buffer from circular queue ++ ret = ff_v4l2_request_reset_picture(avctx, pic); + if (ret) + return ret; + -+ return v4l2_request_h264_set_controls(avctx); ++ // Ensure capture buffer is dequeued before reuse ++ ret = v4l2_request_wait_on_capture(ctx, &desc->capture); ++ if (ret) ++ return ret; ++ ++ // Wait on capture buffer in post_process() before returning to application ++ fdd->hwaccel_priv = ctx; ++ fdd->post_process = v4l2_request_post_process; ++ ++ // Capture buffer used for current frame ++ pic->capture = &desc->capture; ++ ++ return 0; +} + -+const AVHWAccel ff_h264_v4l2request_hwaccel = { -+ .name = "h264_v4l2request", -+ .type = AVMEDIA_TYPE_VIDEO, -+ .id = AV_CODEC_ID_H264, -+ .pix_fmt = AV_PIX_FMT_DRM_PRIME, -+ .start_frame = v4l2_request_h264_start_frame, -+ .decode_slice = v4l2_request_h264_decode_slice, -+ .end_frame = v4l2_request_h264_end_frame, -+ .frame_priv_data_size = sizeof(V4L2RequestControlsH264), -+ .init = v4l2_request_h264_init, -+ .uninit = ff_v4l2_request_uninit, -+ .priv_data_size = sizeof(V4L2RequestContextH264), -+ .frame_params = ff_v4l2_request_frame_params, -+ .caps_internal = HWACCEL_CAP_ASYNC_SAFE, -+}; ++void ff_v4l2_request_flush(AVCodecContext *avctx) ++{ ++ V4L2RequestContext *ctx = v4l2_request_context(avctx); ++ struct pollfd pollfd = { ++ .fd = ctx->video_fd, ++ .events = POLLOUT, ++ }; ++ ++ ff_mutex_lock(&ctx->mutex); ++ ++ // Dequeue all completed output buffers ++ if (atomic_load(&ctx->queued_output)) ++ v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type); ++ ++ // Wait on any remaining output buffer ++ while (atomic_load(&ctx->queued_output)) { ++ int ret = poll(&pollfd, 1, 2000); ++ if (ret <= 0) ++ break; ++ ++ ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type); ++ if (ret < 0 && ret != AVERROR(EAGAIN)) ++ break; ++ } ++ ++ // Dequeue all completed capture buffers ++ if (atomic_load(&ctx->queued_capture)) ++ v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type); ++ ++ ff_mutex_unlock(&ctx->mutex); ++} -From f097a36c1796aba5b5be011ae124cea150cce1f9 Mon Sep 17 00:00:00 2001 +From 66cb225496620a953033096eb1013692be9cb931 Mon Sep 17 00:00:00 2001 From: Jonas Karlman -Date: Sat, 15 Dec 2018 22:32:16 +0100 -Subject: [PATCH 05/11] Add V4L2 request API mpeg2 hwaccel +Date: Tue, 6 Aug 2024 09:06:04 +0000 +Subject: [PATCH 05/11] avcodec: Add V4L2 Request API mpeg2 hwaccel + +Add a V4L2 Request API hwaccel for MPEG2. + +Support for MPEG2 is enabled when Linux kernel headers declare the +control id V4L2_CID_STATELESS_MPEG2_SEQUENCE, added in v5.14. + +This also change v4l2_request hwaccel to use autodetect in configure. Signed-off-by: Jonas Karlman --- - configure | 3 + + configure | 7 +- libavcodec/Makefile | 1 + libavcodec/hwaccels.h | 1 + libavcodec/mpeg12dec.c | 6 ++ - libavcodec/v4l2_request_mpeg2.c | 159 ++++++++++++++++++++++++++++++++ - 5 files changed, 170 insertions(+) + libavcodec/v4l2_request_mpeg2.c | 176 ++++++++++++++++++++++++++++++++ + 5 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 libavcodec/v4l2_request_mpeg2.c diff --git a/configure b/configure -index 8a475352c2ca..b38288656cc4 100755 +index b74da32e8b..cbf0ee0ced 100755 --- a/configure +++ b/configure -@@ -3073,6 +3073,8 @@ mpeg2_dxva2_hwaccel_deps="dxva2" +@@ -359,7 +359,7 @@ External library support: + --enable-omx-rpi enable OpenMAX IL code for Raspberry Pi [no] + --enable-rkmpp enable Rockchip Media Process Platform code [no] + --disable-v4l2-m2m disable V4L2 mem2mem code [autodetect] +- --enable-v4l2-request enable V4L2 Request API code [no] ++ --disable-v4l2-request disable V4L2 Request API code [autodetect] + --disable-vaapi disable Video Acceleration API (mainly Unix/Intel) code [autodetect] + --disable-vdpau disable Nvidia Video Decode and Presentation API for Unix code [autodetect] + --disable-videotoolbox disable VideoToolbox code [autodetect] +@@ -2006,6 +2006,7 @@ HWACCEL_AUTODETECT_LIBRARY_LIST=" + videotoolbox + vulkan + v4l2_m2m ++ v4l2_request + " + + # catchall list of things that require external libs to link +@@ -2027,7 +2028,6 @@ HWACCEL_LIBRARY_LIST=" + mmal + omx + opencl +- v4l2_request + " + + DOCUMENT_LIST=" +@@ -3232,6 +3232,8 @@ mpeg2_dxva2_hwaccel_deps="dxva2" mpeg2_dxva2_hwaccel_select="mpeg2video_decoder" mpeg2_nvdec_hwaccel_deps="nvdec" mpeg2_nvdec_hwaccel_select="mpeg2video_decoder" @@ -1976,20 +2540,20 @@ index 8a475352c2ca..b38288656cc4 100755 mpeg2_vaapi_hwaccel_deps="vaapi" mpeg2_vaapi_hwaccel_select="mpeg2video_decoder" mpeg2_vdpau_hwaccel_deps="vdpau" -@@ -6890,6 +6892,7 @@ fi +@@ -7202,6 +7204,7 @@ if enabled v4l2_m2m; then + fi - check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns - check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_H264_SLICE;" -+check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_MPEG2_SLICE;" - - check_headers sys/videoio.h - test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete + if enabled v4l2_request; then ++ check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" + check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns + check_pkg_config libudev libudev libudev.h udev_new diff --git a/libavcodec/Makefile b/libavcodec/Makefile -index 26004171b31a..47cc14558c49 100644 +index 6f2229b1b6..2c69b747b1 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile -@@ -1009,6 +1009,7 @@ OBJS-$(CONFIG_MPEG2_D3D11VA_HWACCEL) += dxva2_mpeg2.o - OBJS-$(CONFIG_MPEG2_DXVA2_HWACCEL) += dxva2_mpeg2.o +@@ -1039,6 +1039,7 @@ OBJS-$(CONFIG_MPEG2_DXVA2_HWACCEL) += dxva2_mpeg2.o + OBJS-$(CONFIG_MPEG2_D3D12VA_HWACCEL) += dxva2_mpeg2.o d3d12va_mpeg2.o OBJS-$(CONFIG_MPEG2_NVDEC_HWACCEL) += nvdec_mpeg12.o OBJS-$(CONFIG_MPEG2_QSV_HWACCEL) += qsvdec.o +OBJS-$(CONFIG_MPEG2_V4L2REQUEST_HWACCEL) += v4l2_request_mpeg2.o @@ -1997,22 +2561,22 @@ index 26004171b31a..47cc14558c49 100644 OBJS-$(CONFIG_MPEG2_VDPAU_HWACCEL) += vdpau_mpeg12.o OBJS-$(CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h -index 014b95f0c05d..3b675dd9f8de 100644 +index 5171e4c7d7..0cba7c71be 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h -@@ -53,6 +53,7 @@ extern const AVHWAccel ff_mpeg2_d3d11va_hwaccel; - extern const AVHWAccel ff_mpeg2_d3d11va2_hwaccel; - extern const AVHWAccel ff_mpeg2_nvdec_hwaccel; - extern const AVHWAccel ff_mpeg2_dxva2_hwaccel; -+extern const AVHWAccel ff_mpeg2_v4l2request_hwaccel; - extern const AVHWAccel ff_mpeg2_vaapi_hwaccel; - extern const AVHWAccel ff_mpeg2_vdpau_hwaccel; - extern const AVHWAccel ff_mpeg2_videotoolbox_hwaccel; +@@ -57,6 +57,7 @@ extern const struct FFHWAccel ff_mpeg2_d3d11va2_hwaccel; + extern const struct FFHWAccel ff_mpeg2_d3d12va_hwaccel; + extern const struct FFHWAccel ff_mpeg2_dxva2_hwaccel; + extern const struct FFHWAccel ff_mpeg2_nvdec_hwaccel; ++extern const struct FFHWAccel ff_mpeg2_v4l2request_hwaccel; + extern const struct FFHWAccel ff_mpeg2_vaapi_hwaccel; + extern const struct FFHWAccel ff_mpeg2_vdpau_hwaccel; + extern const struct FFHWAccel ff_mpeg2_videotoolbox_hwaccel; diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c -index 6c4cbccc597c..1bf6b87a6b62 100644 +index 4f784611de..af8ffa5976 100644 --- a/libavcodec/mpeg12dec.c +++ b/libavcodec/mpeg12dec.c -@@ -1134,6 +1134,9 @@ static const enum AVPixelFormat mpeg2_hwaccel_pixfmt_list_420[] = { +@@ -839,6 +839,9 @@ static const enum AVPixelFormat mpeg2_hwaccel_pixfmt_list_420[] = { #endif #if CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL AV_PIX_FMT_VIDEOTOOLBOX, @@ -2022,7 +2586,7 @@ index 6c4cbccc597c..1bf6b87a6b62 100644 #endif AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE -@@ -2919,6 +2922,9 @@ const FFCodec ff_mpeg2video_decoder = { +@@ -2681,6 +2684,9 @@ const FFCodec ff_mpeg2video_decoder = { #endif #if CONFIG_MPEG2_VIDEOTOOLBOX_HWACCEL HWACCEL_VIDEOTOOLBOX(mpeg2), @@ -2034,10 +2598,10 @@ index 6c4cbccc597c..1bf6b87a6b62 100644 }, diff --git a/libavcodec/v4l2_request_mpeg2.c b/libavcodec/v4l2_request_mpeg2.c new file mode 100644 -index 000000000000..84d53209c79d +index 0000000000..998c91355a --- /dev/null +++ b/libavcodec/v4l2_request_mpeg2.c -@@ -0,0 +1,159 @@ +@@ -0,0 +1,176 @@ +/* + * This file is part of FFmpeg. + * @@ -2056,11 +2620,15 @@ index 000000000000..84d53209c79d + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + ++#include "config.h" ++ ++#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "mpegvideo.h" +#include "v4l2_request.h" + +typedef struct V4L2RequestControlsMPEG2 { ++ V4L2RequestPictureContext pic; + struct v4l2_ctrl_mpeg2_sequence sequence; + struct v4l2_ctrl_mpeg2_picture picture; + struct v4l2_ctrl_mpeg2_quantisation quantisation; @@ -2071,14 +2639,18 @@ index 000000000000..84d53209c79d + av_unused uint32_t size) +{ + const MpegEncContext *s = avctx->priv_data; -+ V4L2RequestControlsMPEG2 *controls = s->current_picture_ptr->hwaccel_picture_private; -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)s->current_picture_ptr->f->data[0]; ++ V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private; ++ int ret; ++ ++ ret = ff_v4l2_request_start_frame(avctx, &controls->pic, s->cur_pic.ptr->f); ++ if (ret) ++ return ret; + + controls->sequence = (struct v4l2_ctrl_mpeg2_sequence) { + /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence header */ + .horizontal_size = s->width, + .vertical_size = s->height, -+ .vbv_buffer_size = req->output.size, ++ .vbv_buffer_size = controls->pic.output->size, + + /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence extension */ + .profile_and_level_indication = 0, @@ -2127,10 +2699,14 @@ index 000000000000..84d53209c79d + + switch (s->pict_type) { + case AV_PICTURE_TYPE_B: -+ controls->picture.backward_ref_ts = ff_v4l2_request_get_capture_timestamp(s->next_picture.f); ++ if (s->next_pic.ptr) ++ controls->picture.backward_ref_ts = ++ ff_v4l2_request_get_capture_timestamp(s->next_pic.ptr->f); + // fall-through + case AV_PICTURE_TYPE_P: -+ controls->picture.forward_ref_ts = ff_v4l2_request_get_capture_timestamp(s->last_picture.f); ++ if (s->last_pic.ptr) ++ controls->picture.forward_ref_ts = ++ ff_v4l2_request_get_capture_timestamp(s->last_pic.ptr->f); + } + + for (int i = 0; i < 64; i++) { @@ -2141,20 +2717,22 @@ index 000000000000..84d53209c79d + controls->quantisation.chroma_non_intra_quantiser_matrix[i] = s->chroma_inter_matrix[n]; + } + -+ return ff_v4l2_request_reset_frame(avctx, s->current_picture_ptr->f); ++ return 0; +} + -+static int v4l2_request_mpeg2_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) ++static int v4l2_request_mpeg2_decode_slice(AVCodecContext *avctx, ++ const uint8_t *buffer, uint32_t size) +{ + const MpegEncContext *s = avctx->priv_data; ++ V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private; + -+ return ff_v4l2_request_append_output_buffer(avctx, s->current_picture_ptr->f, buffer, size); ++ return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); +} + +static int v4l2_request_mpeg2_end_frame(AVCodecContext *avctx) +{ + const MpegEncContext *s = avctx->priv_data; -+ V4L2RequestControlsMPEG2 *controls = s->current_picture_ptr->hwaccel_picture_private; ++ V4L2RequestControlsMPEG2 *controls = s->cur_pic.ptr->hwaccel_picture_private; + + struct v4l2_ext_control control[] = { + { @@ -2174,98 +2752,242 @@ index 000000000000..84d53209c79d + }, + }; + -+ return ff_v4l2_request_decode_frame(avctx, s->current_picture_ptr->f, control, FF_ARRAY_ELEMS(control)); ++ return ff_v4l2_request_decode_frame(avctx, &controls->pic, ++ control, FF_ARRAY_ELEMS(control)); +} + +static int v4l2_request_mpeg2_init(AVCodecContext *avctx) +{ -+ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_MPEG2_SLICE, 1024 * 1024, NULL, 0); ++ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_MPEG2_SLICE, ++ 1024 * 1024, ++ NULL, 0); +} + -+const AVHWAccel ff_mpeg2_v4l2request_hwaccel = { -+ .name = "mpeg2_v4l2request", -+ .type = AVMEDIA_TYPE_VIDEO, -+ .id = AV_CODEC_ID_MPEG2VIDEO, -+ .pix_fmt = AV_PIX_FMT_DRM_PRIME, -+ .start_frame = v4l2_request_mpeg2_start_frame, -+ .decode_slice = v4l2_request_mpeg2_decode_slice, -+ .end_frame = v4l2_request_mpeg2_end_frame, ++const FFHWAccel ff_mpeg2_v4l2request_hwaccel = { ++ .p.name = "mpeg2_v4l2request", ++ .p.type = AVMEDIA_TYPE_VIDEO, ++ .p.id = AV_CODEC_ID_MPEG2VIDEO, ++ .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, ++ .start_frame = v4l2_request_mpeg2_start_frame, ++ .decode_slice = v4l2_request_mpeg2_decode_slice, ++ .end_frame = v4l2_request_mpeg2_end_frame, ++ .flush = ff_v4l2_request_flush, + .frame_priv_data_size = sizeof(V4L2RequestControlsMPEG2), -+ .init = v4l2_request_mpeg2_init, -+ .uninit = ff_v4l2_request_uninit, -+ .priv_data_size = sizeof(V4L2RequestContext), -+ .frame_params = ff_v4l2_request_frame_params, -+ .caps_internal = HWACCEL_CAP_ASYNC_SAFE, ++ .init = v4l2_request_mpeg2_init, ++ .uninit = ff_v4l2_request_uninit, ++ .priv_data_size = sizeof(V4L2RequestContext), ++ .frame_params = ff_v4l2_request_frame_params, +}; -From bcd97d231b9d9da9a5dabe2c4c46a8aedc1a9576 Mon Sep 17 00:00:00 2001 +From bccaf9b66ca4e1180c0d42bc95daf540e1e2ffb7 Mon Sep 17 00:00:00 2001 From: Boris Brezillon -Date: Wed, 22 May 2019 14:46:58 +0200 -Subject: [PATCH 06/11] Add V4L2 request API vp8 hwaccel +Date: Tue, 6 Aug 2024 09:06:05 +0000 +Subject: [PATCH 06/11] avcodec/h264dec: add ref_pic_marking and pic_order_cnt + bit_size to slice context + +The V4L2_CID_STATELESS_H264_DECODE_PARAMS control require following: + +- dec_ref_pic_marking_bit_size + Size in bits of the dec_ref_pic_marking() syntax element. + +- pic_order_cnt_bit_size + Combined size in bits of the picture order count related syntax + elements: pic_order_cnt_lsb, delta_pic_order_cnt_bottom, + delta_pic_order_cnt0, and delta_pic_order_cnt1. + +Save the bit sizes while parsing for later use in hwaccel, similar to +short/long_term_ref_pic_set_size in hevcdec. Signed-off-by: Boris Brezillon -Signed-off-by: Ezequiel Garcia Signed-off-by: Jonas Karlman --- - configure | 3 + - libavcodec/Makefile | 1 + - libavcodec/hwaccels.h | 1 + - libavcodec/v4l2_request_vp8.c | 180 ++++++++++++++++++++++++++++++++++ - libavcodec/vp8.c | 6 ++ - 5 files changed, 191 insertions(+) - create mode 100644 libavcodec/v4l2_request_vp8.c + libavcodec/h264_slice.c | 6 +++++- + libavcodec/h264dec.h | 2 ++ + 2 files changed, 7 insertions(+), 1 deletion(-) + +diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c +index a66b75ca80..58a48f3fbe 100644 +--- a/libavcodec/h264_slice.c ++++ b/libavcodec/h264_slice.c +@@ -1690,7 +1690,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, + unsigned int slice_type, tmp, i; + int field_pic_flag, bottom_field_flag; + int first_slice = sl == h->slice_ctx && !h->current_slice; +- int picture_structure; ++ int picture_structure, pos; + + if (first_slice) + av_assert0(!h->setup_finished); +@@ -1781,6 +1781,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, + + sl->poc_lsb = 0; + sl->delta_poc_bottom = 0; ++ pos = get_bits_left(&sl->gb); + if (sps->poc_type == 0) { + sl->poc_lsb = get_bits(&sl->gb, sps->log2_max_poc_lsb); + +@@ -1795,6 +1796,7 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, + if (pps->pic_order_present == 1 && picture_structure == PICT_FRAME) + sl->delta_poc[1] = get_se_golomb(&sl->gb); + } ++ sl->pic_order_cnt_bit_size = pos - get_bits_left(&sl->gb); + + sl->redundant_pic_count = 0; + if (pps->redundant_pic_cnt_present) +@@ -1834,9 +1836,11 @@ static int h264_slice_header_parse(const H264Context *h, H264SliceContext *sl, + + sl->explicit_ref_marking = 0; + if (nal->ref_idc) { ++ pos = get_bits_left(&sl->gb); + ret = ff_h264_decode_ref_pic_marking(sl, &sl->gb, nal, h->avctx); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) + return AVERROR_INVALIDDATA; ++ sl->ref_pic_marking_bit_size = pos - get_bits_left(&sl->gb); + } + + if (sl->slice_type_nos != AV_PICTURE_TYPE_I && pps->cabac) { +diff --git a/libavcodec/h264dec.h b/libavcodec/h264dec.h +index ccd7583bf4..71cbb0e0cc 100644 +--- a/libavcodec/h264dec.h ++++ b/libavcodec/h264dec.h +@@ -324,6 +324,7 @@ typedef struct H264SliceContext { + MMCO mmco[H264_MAX_MMCO_COUNT]; + int nb_mmco; + int explicit_ref_marking; ++ int ref_pic_marking_bit_size; + + int frame_num; + int idr_pic_id; +@@ -332,6 +333,7 @@ typedef struct H264SliceContext { + int delta_poc[2]; + int curr_pic_num; + int max_pic_num; ++ int pic_order_cnt_bit_size; + } H264SliceContext; + + /** + +From e7674a31a7bc8b37a8ba4be0fc59074333a95b2d Mon Sep 17 00:00:00 2001 +From: Jernej Skrabec +Date: Tue, 6 Aug 2024 09:06:06 +0000 +Subject: [PATCH 07/11] avcodec: Add V4L2 Request API h264 hwaccel + +Add a V4L2 Request API hwaccel for H.264, supporting both slice and +frame decoding modes. + +Support for H.264 is enabled when Linux kernel headers declare the +control id V4L2_CID_STATELESS_H264_DECODE_MODE, added in v5.11. + +Signed-off-by: Jernej Skrabec +Co-developed-by: Jonas Karlman +Signed-off-by: Jonas Karlman +--- + configure | 3 + + libavcodec/Makefile | 1 + + libavcodec/h264_slice.c | 7 + + libavcodec/h264dec.c | 3 + + libavcodec/hwaccels.h | 1 + + libavcodec/v4l2_request_h264.c | 522 +++++++++++++++++++++++++++++++++ + 6 files changed, 537 insertions(+) + create mode 100644 libavcodec/v4l2_request_h264.c diff --git a/configure b/configure -index b38288656cc4..d6fd3281a38c 100755 +index cbf0ee0ced..6866a61153 100755 --- a/configure +++ b/configure -@@ -3105,6 +3105,8 @@ vc1_vdpau_hwaccel_deps="vdpau" - vc1_vdpau_hwaccel_select="vc1_decoder" - vp8_nvdec_hwaccel_deps="nvdec" - vp8_nvdec_hwaccel_select="vp8_decoder" -+vp8_v4l2request_hwaccel_deps="v4l2_request vp8_v4l2_request" -+vp8_v4l2request_hwaccel_select="vp8_decoder" - vp8_vaapi_hwaccel_deps="vaapi" - vp8_vaapi_hwaccel_select="vp8_decoder" - vp9_d3d11va_hwaccel_deps="d3d11va DXVA_PicParams_VP9" -@@ -6893,6 +6895,7 @@ fi - check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns - check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_H264_SLICE;" - check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_MPEG2_SLICE;" -+check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_VP8_FRAME;" +@@ -3186,6 +3186,8 @@ h264_dxva2_hwaccel_deps="dxva2" + h264_dxva2_hwaccel_select="h264_decoder" + h264_nvdec_hwaccel_deps="nvdec" + h264_nvdec_hwaccel_select="h264_decoder" ++h264_v4l2request_hwaccel_deps="v4l2_request h264_v4l2_request" ++h264_v4l2request_hwaccel_select="h264_decoder" + h264_vaapi_hwaccel_deps="vaapi" + h264_vaapi_hwaccel_select="h264_decoder" + h264_vdpau_hwaccel_deps="vdpau" +@@ -7204,6 +7206,7 @@ if enabled v4l2_m2m; then + fi - check_headers sys/videoio.h - test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete + if enabled v4l2_request; then ++ check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE" + check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" + check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns diff --git a/libavcodec/Makefile b/libavcodec/Makefile -index 47cc14558c49..7da4fd1a8718 100644 +index 2c69b747b1..00bc1cf8f2 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile -@@ -1024,6 +1024,7 @@ OBJS-$(CONFIG_VC1_QSV_HWACCEL) += qsvdec.o - OBJS-$(CONFIG_VC1_VAAPI_HWACCEL) += vaapi_vc1.o - OBJS-$(CONFIG_VC1_VDPAU_HWACCEL) += vdpau_vc1.o - OBJS-$(CONFIG_VP8_NVDEC_HWACCEL) += nvdec_vp8.o -+OBJS-$(CONFIG_VP8_V4L2REQUEST_HWACCEL) += v4l2_request_vp8.o - OBJS-$(CONFIG_VP8_VAAPI_HWACCEL) += vaapi_vp8.o - OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL) += dxva2_vp9.o - OBJS-$(CONFIG_VP9_DXVA2_HWACCEL) += dxva2_vp9.o +@@ -1017,6 +1017,7 @@ OBJS-$(CONFIG_H264_DXVA2_HWACCEL) += dxva2_h264.o + OBJS-$(CONFIG_H264_D3D12VA_HWACCEL) += dxva2_h264.o d3d12va_h264.o + OBJS-$(CONFIG_H264_NVDEC_HWACCEL) += nvdec_h264.o + OBJS-$(CONFIG_H264_QSV_HWACCEL) += qsvdec.o ++OBJS-$(CONFIG_H264_V4L2REQUEST_HWACCEL) += v4l2_request_h264.o + OBJS-$(CONFIG_H264_VAAPI_HWACCEL) += vaapi_h264.o + OBJS-$(CONFIG_H264_VDPAU_HWACCEL) += vdpau_h264.o + OBJS-$(CONFIG_H264_VIDEOTOOLBOX_HWACCEL) += videotoolbox.o +diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c +index 58a48f3fbe..259a2c307a 100644 +--- a/libavcodec/h264_slice.c ++++ b/libavcodec/h264_slice.c +@@ -781,6 +781,7 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) + (CONFIG_H264_D3D11VA_HWACCEL * 2) + \ + CONFIG_H264_D3D12VA_HWACCEL + \ + CONFIG_H264_NVDEC_HWACCEL + \ ++ CONFIG_H264_V4L2REQUEST_HWACCEL + \ + CONFIG_H264_VAAPI_HWACCEL + \ + CONFIG_H264_VIDEOTOOLBOX_HWACCEL + \ + CONFIG_H264_VDPAU_HWACCEL + \ +@@ -806,6 +807,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) + #endif + #if CONFIG_H264_VULKAN_HWACCEL + *fmt++ = AV_PIX_FMT_VULKAN; ++#endif ++#if CONFIG_H264_V4L2REQUEST_HWACCEL ++ *fmt++ = AV_PIX_FMT_DRM_PRIME; + #endif + if (CHROMA444(h)) { + if (h->avctx->colorspace == AVCOL_SPC_RGB) { +@@ -862,6 +866,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) + #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL + if (h->avctx->colorspace != AVCOL_SPC_RGB) + *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX; ++#endif ++#if CONFIG_H264_V4L2REQUEST_HWACCEL ++ *fmt++ = AV_PIX_FMT_DRM_PRIME; + #endif + if (CHROMA444(h)) { + if (h->avctx->colorspace == AVCOL_SPC_RGB) +diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c +index 0154fe17b6..a5e89ebddd 100644 +--- a/libavcodec/h264dec.c ++++ b/libavcodec/h264dec.c +@@ -1160,6 +1160,9 @@ const FFCodec ff_h264_decoder = { + #endif + #if CONFIG_H264_VULKAN_HWACCEL + HWACCEL_VULKAN(h264), ++#endif ++#if CONFIG_H264_V4L2REQUEST_HWACCEL ++ HWACCEL_V4L2REQUEST(h264), + #endif + NULL + }, diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h -index 3b675dd9f8de..6f9f0780019b 100644 +index 0cba7c71be..024e021d73 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h -@@ -69,6 +69,7 @@ extern const AVHWAccel ff_vc1_nvdec_hwaccel; - extern const AVHWAccel ff_vc1_vaapi_hwaccel; - extern const AVHWAccel ff_vc1_vdpau_hwaccel; - extern const AVHWAccel ff_vp8_nvdec_hwaccel; -+extern const AVHWAccel ff_vp8_v4l2request_hwaccel; - extern const AVHWAccel ff_vp8_vaapi_hwaccel; - extern const AVHWAccel ff_vp9_d3d11va_hwaccel; - extern const AVHWAccel ff_vp9_d3d11va2_hwaccel; -diff --git a/libavcodec/v4l2_request_vp8.c b/libavcodec/v4l2_request_vp8.c +@@ -34,6 +34,7 @@ extern const struct FFHWAccel ff_h264_d3d11va2_hwaccel; + extern const struct FFHWAccel ff_h264_d3d12va_hwaccel; + extern const struct FFHWAccel ff_h264_dxva2_hwaccel; + extern const struct FFHWAccel ff_h264_nvdec_hwaccel; ++extern const struct FFHWAccel ff_h264_v4l2request_hwaccel; + extern const struct FFHWAccel ff_h264_vaapi_hwaccel; + extern const struct FFHWAccel ff_h264_vdpau_hwaccel; + extern const struct FFHWAccel ff_h264_videotoolbox_hwaccel; +diff --git a/libavcodec/v4l2_request_h264.c b/libavcodec/v4l2_request_h264.c new file mode 100644 -index 000000000000..e16903021357 +index 0000000000..5affe309ac --- /dev/null -+++ b/libavcodec/v4l2_request_vp8.c -@@ -0,0 +1,180 @@ ++++ b/libavcodec/v4l2_request_h264.c +@@ -0,0 +1,522 @@ +/* + * This file is part of FFmpeg. + * @@ -2284,216 +3006,551 @@ index 000000000000..e16903021357 + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + ++#include "config.h" ++ ++#include "h264dec.h" ++#include "hwaccel_internal.h" +#include "hwconfig.h" ++#include "internal.h" +#include "v4l2_request.h" -+#include "vp8.h" + -+typedef struct V4L2RequestControlsVP8 { -+ struct v4l2_ctrl_vp8_frame ctrl; -+} V4L2RequestControlsVP8; ++typedef struct V4L2RequestContextH264 { ++ V4L2RequestContext base; ++ enum v4l2_stateless_h264_decode_mode decode_mode; ++ enum v4l2_stateless_h264_start_code start_code; ++} V4L2RequestContextH264; + -+static int v4l2_request_vp8_start_frame(AVCodecContext *avctx, -+ av_unused const uint8_t *buffer, -+ av_unused uint32_t size) ++typedef struct V4L2RequestControlsH264 { ++ V4L2RequestPictureContext pic; ++ struct v4l2_ctrl_h264_sps sps; ++ struct v4l2_ctrl_h264_pps pps; ++ struct v4l2_ctrl_h264_scaling_matrix scaling_matrix; ++ struct v4l2_ctrl_h264_decode_params decode_params; ++ struct v4l2_ctrl_h264_slice_params slice_params; ++ struct v4l2_ctrl_h264_pred_weights pred_weights; ++ bool pred_weights_required; ++ bool first_slice; ++ int num_slices; ++} V4L2RequestControlsH264; ++ ++static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 }; ++ ++static void fill_weight_factors(struct v4l2_h264_weight_factors *weight_factors, ++ int list, const H264SliceContext *sl) +{ -+ const VP8Context *s = avctx->priv_data; -+ V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; -+ -+ memset(&controls->ctrl, 0, sizeof(controls->ctrl)); -+ return ff_v4l2_request_reset_frame(avctx, s->framep[VP8_FRAME_CURRENT]->tf.f); -+} -+ -+static int v4l2_request_vp8_end_frame(AVCodecContext *avctx) -+{ -+ const VP8Context *s = avctx->priv_data; -+ V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; -+ struct v4l2_ext_control control[] = { -+ { -+ .id = V4L2_CID_STATELESS_VP8_FRAME, -+ .ptr = &controls->ctrl, -+ .size = sizeof(controls->ctrl), -+ }, -+ }; -+ -+ return ff_v4l2_request_decode_frame(avctx, s->framep[VP8_FRAME_CURRENT]->tf.f, -+ control, FF_ARRAY_ELEMS(control)); -+} -+ -+static int v4l2_request_vp8_decode_slice(AVCodecContext *avctx, -+ const uint8_t *buffer, -+ uint32_t size) -+{ -+ const VP8Context *s = avctx->priv_data; -+ V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; -+ struct v4l2_ctrl_vp8_frame *frame = &controls->ctrl; -+ const uint8_t *data = buffer + 3 + 7 * s->keyframe; -+ unsigned int i, j, k; -+ -+ frame->version = s->profile & 0x3; -+ frame->width = avctx->width; -+ frame->height = avctx->height; -+ /* FIXME: set ->xx_scale */ -+ frame->prob_skip_false = s->prob->mbskip; -+ frame->prob_intra = s->prob->intra; -+ frame->prob_gf = s->prob->golden; -+ frame->prob_last = s->prob->last; -+ frame->first_part_size = s->header_partition_size; -+ frame->first_part_header_bits = (8 * (s->coder_state_at_header_end.input - data) - -+ s->coder_state_at_header_end.bit_count - 8); -+ frame->num_dct_parts = s->num_coeff_partitions; -+ for (i = 0; i < 8; i++) -+ frame->dct_part_sizes[i] = s->coeff_partition_size[i]; -+ -+ frame->coder_state.range = s->coder_state_at_header_end.range; -+ frame->coder_state.value = s->coder_state_at_header_end.value; -+ frame->coder_state.bit_count = s->coder_state_at_header_end.bit_count; -+ if (s->framep[VP8_FRAME_PREVIOUS]) -+ frame->last_frame_ts = ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_PREVIOUS]->tf.f); -+ if (s->framep[VP8_FRAME_GOLDEN]) -+ frame->golden_frame_ts = ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_GOLDEN]->tf.f); -+ if (s->framep[VP8_FRAME_ALTREF]) -+ frame->alt_frame_ts = ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_ALTREF]->tf.f); -+ frame->flags |= s->invisible ? 0 : V4L2_VP8_FRAME_FLAG_SHOW_FRAME; -+ frame->flags |= s->mbskip_enabled ? V4L2_VP8_FRAME_FLAG_MB_NO_SKIP_COEFF : 0; -+ frame->flags |= (s->profile & 0x4) ? V4L2_VP8_FRAME_FLAG_EXPERIMENTAL : 0; -+ frame->flags |= s->keyframe ? V4L2_VP8_FRAME_FLAG_KEY_FRAME : 0; -+ frame->flags |= s->sign_bias[VP8_FRAME_GOLDEN] ? V4L2_VP8_FRAME_FLAG_SIGN_BIAS_GOLDEN : 0; -+ frame->flags |= s->sign_bias[VP8_FRAME_ALTREF] ? V4L2_VP8_FRAME_FLAG_SIGN_BIAS_ALT : 0; -+ frame->segment.flags |= s->segmentation.enabled ? V4L2_VP8_SEGMENT_FLAG_ENABLED : 0; -+ frame->segment.flags |= s->segmentation.update_map ? V4L2_VP8_SEGMENT_FLAG_UPDATE_MAP : 0; -+ frame->segment.flags |= s->segmentation.update_feature_data ? V4L2_VP8_SEGMENT_FLAG_UPDATE_FEATURE_DATA : 0; -+ frame->segment.flags |= s->segmentation.absolute_vals ? 0 : V4L2_VP8_SEGMENT_FLAG_DELTA_VALUE_MODE; -+ for (i = 0; i < 4; i++) { -+ frame->segment.quant_update[i] = s->segmentation.base_quant[i]; -+ frame->segment.lf_update[i] = s->segmentation.filter_level[i]; -+ } -+ -+ for (i = 0; i < 3; i++) -+ frame->segment.segment_probs[i] = s->prob->segmentid[i]; -+ -+ frame->lf.level = s->filter.level; -+ frame->lf.sharpness_level = s->filter.sharpness; -+ frame->lf.flags |= s->lf_delta.enabled ? V4L2_VP8_LF_ADJ_ENABLE : 0; -+ frame->lf.flags |= s->lf_delta.update ? V4L2_VP8_LF_DELTA_UPDATE : 0; -+ frame->lf.flags |= s->filter.simple ? V4L2_VP8_LF_FILTER_TYPE_SIMPLE : 0; -+ for (i = 0; i < 4; i++) { -+ frame->lf.ref_frm_delta[i] = s->lf_delta.ref[i]; -+ frame->lf.mb_mode_delta[i] = s->lf_delta.mode[i + MODE_I4x4]; -+ } -+ -+ // Probabilites -+ if (s->keyframe) { -+ static const uint8_t keyframe_y_mode_probs[4] = { -+ 145, 156, 163, 128 -+ }; -+ static const uint8_t keyframe_uv_mode_probs[3] = { -+ 142, 114, 183 -+ }; -+ -+ memcpy(frame->entropy.y_mode_probs, keyframe_y_mode_probs, 4); -+ memcpy(frame->entropy.uv_mode_probs, keyframe_uv_mode_probs, 3); -+ } else { -+ for (i = 0; i < 4; i++) -+ frame->entropy.y_mode_probs[i] = s->prob->pred16x16[i]; -+ for (i = 0; i < 3; i++) -+ frame->entropy.uv_mode_probs[i] = s->prob->pred8x8c[i]; -+ } -+ for (i = 0; i < 2; i++) -+ for (j = 0; j < 19; j++) -+ frame->entropy.mv_probs[i][j] = s->prob->mvc[i][j]; -+ -+ for (i = 0; i < 4; i++) { -+ for (j = 0; j < 8; j++) { -+ static const int coeff_bands_inverse[8] = { -+ 0, 1, 2, 3, 5, 6, 4, 15 -+ }; -+ int coeff_pos = coeff_bands_inverse[j]; -+ -+ for (k = 0; k < 3; k++) { -+ memcpy(frame->entropy.coeff_probs[i][j][k], -+ s->prob->token[i][coeff_pos][k], 11); ++ for (int i = 0; i < sl->ref_count[list]; i++) { ++ if (sl->pwt.luma_weight_flag[list]) { ++ weight_factors->luma_weight[i] = sl->pwt.luma_weight[i][list][0]; ++ weight_factors->luma_offset[i] = sl->pwt.luma_weight[i][list][1]; ++ } else { ++ weight_factors->luma_weight[i] = 1 << sl->pwt.luma_log2_weight_denom; ++ weight_factors->luma_offset[i] = 0; ++ } ++ for (int j = 0; j < 2; j++) { ++ if (sl->pwt.chroma_weight_flag[list]) { ++ weight_factors->chroma_weight[i][j] = sl->pwt.chroma_weight[i][list][j][0]; ++ weight_factors->chroma_offset[i][j] = sl->pwt.chroma_weight[i][list][j][1]; ++ } else { ++ weight_factors->chroma_weight[i][j] = 1 << sl->pwt.chroma_log2_weight_denom; ++ weight_factors->chroma_offset[i][j] = 0; + } + } + } -+ -+ frame->quant.y_ac_qi = s->quant.yac_qi; -+ frame->quant.y_dc_delta = s->quant.ydc_delta; -+ frame->quant.y2_dc_delta = s->quant.y2dc_delta; -+ frame->quant.y2_ac_delta = s->quant.y2ac_delta; -+ frame->quant.uv_dc_delta = s->quant.uvdc_delta; -+ frame->quant.uv_ac_delta = s->quant.uvac_delta; -+ -+ return ff_v4l2_request_append_output_buffer(avctx, s->framep[VP8_FRAME_CURRENT]->tf.f, buffer, size); +} + -+static int v4l2_request_vp8_init(AVCodecContext *avctx) ++static void fill_dpb_entry(struct v4l2_h264_dpb_entry *entry, ++ const H264Picture *pic, int long_idx) +{ -+ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_VP8_FRAME, 2 * 1024 * 1024, NULL, 0); ++ entry->reference_ts = ff_v4l2_request_get_capture_timestamp(pic->f); ++ entry->pic_num = pic->pic_id; ++ entry->frame_num = pic->long_ref ? long_idx : pic->frame_num; ++ entry->fields = pic->reference & V4L2_H264_FRAME_REF; ++ entry->flags = V4L2_H264_DPB_ENTRY_FLAG_VALID; ++ if (entry->fields) ++ entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; ++ if (pic->long_ref) ++ entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM; ++ if (pic->field_picture) ++ entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD; ++ if (pic->field_poc[0] != INT_MAX) ++ entry->top_field_order_cnt = pic->field_poc[0]; ++ if (pic->field_poc[1] != INT_MAX) ++ entry->bottom_field_order_cnt = pic->field_poc[1]; +} + -+const AVHWAccel ff_vp8_v4l2request_hwaccel = { -+ .name = "vp8_v4l2request", -+ .type = AVMEDIA_TYPE_VIDEO, -+ .id = AV_CODEC_ID_VP8, -+ .pix_fmt = AV_PIX_FMT_DRM_PRIME, -+ .start_frame = v4l2_request_vp8_start_frame, -+ .decode_slice = v4l2_request_vp8_decode_slice, -+ .end_frame = v4l2_request_vp8_end_frame, -+ .frame_priv_data_size = sizeof(V4L2RequestControlsVP8), -+ .init = v4l2_request_vp8_init, -+ .uninit = ff_v4l2_request_uninit, -+ .priv_data_size = sizeof(V4L2RequestContext), -+ .frame_params = ff_v4l2_request_frame_params, -+ .caps_internal = HWACCEL_CAP_ASYNC_SAFE, ++static void fill_dpb(struct v4l2_ctrl_h264_decode_params *decode_params, ++ const H264Context *h) ++{ ++ int entries = 0; ++ ++ for (int i = 0; i < h->short_ref_count; i++) { ++ const H264Picture *pic = h->short_ref[i]; ++ if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) ++ fill_dpb_entry(&decode_params->dpb[entries++], pic, pic->pic_id); ++ } ++ ++ if (!h->long_ref_count) ++ return; ++ ++ for (int i = 0; i < FF_ARRAY_ELEMS(h->long_ref); i++) { ++ const H264Picture *pic = h->long_ref[i]; ++ if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) ++ fill_dpb_entry(&decode_params->dpb[entries++], pic, i); ++ } ++} ++ ++static void fill_ref_list(struct v4l2_h264_reference *reference, ++ struct v4l2_ctrl_h264_decode_params *decode_params, ++ const H264Ref *ref) ++{ ++ uint64_t timestamp; ++ ++ if (!ref->parent) ++ return; ++ ++ timestamp = ff_v4l2_request_get_capture_timestamp(ref->parent->f); ++ ++ for (uint8_t i = 0; i < FF_ARRAY_ELEMS(decode_params->dpb); i++) { ++ struct v4l2_h264_dpb_entry *entry = &decode_params->dpb[i]; ++ if ((entry->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) && ++ entry->reference_ts == timestamp) { ++ reference->fields = ref->reference & V4L2_H264_FRAME_REF; ++ reference->index = i; ++ return; ++ } ++ } ++} ++ ++static void fill_sps(struct v4l2_ctrl_h264_sps *ctrl, const H264Context *h) ++{ ++ const SPS *sps = h->ps.sps; ++ ++ *ctrl = (struct v4l2_ctrl_h264_sps) { ++ .profile_idc = sps->profile_idc, ++ .constraint_set_flags = sps->constraint_set_flags, ++ .level_idc = sps->level_idc, ++ .seq_parameter_set_id = sps->sps_id, ++ .chroma_format_idc = sps->chroma_format_idc, ++ .bit_depth_luma_minus8 = sps->bit_depth_luma - 8, ++ .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8, ++ .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4, ++ .pic_order_cnt_type = sps->poc_type, ++ .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4, ++ .max_num_ref_frames = sps->ref_frame_count, ++ .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length, ++ .offset_for_non_ref_pic = sps->offset_for_non_ref_pic, ++ .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field, ++ .pic_width_in_mbs_minus1 = h->mb_width - 1, ++ .pic_height_in_map_units_minus1 = sps->frame_mbs_only_flag ? ++ h->mb_height - 1 : h->mb_height / 2 - 1, ++ }; ++ ++ if (sps->poc_cycle_length > 0 && sps->poc_cycle_length <= 255) ++ memcpy(ctrl->offset_for_ref_frame, sps->offset_for_ref_frame, ++ sps->poc_cycle_length * sizeof(ctrl->offset_for_ref_frame[0])); ++ ++ if (sps->residual_color_transform_flag) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; ++ ++ if (sps->transform_bypass) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; ++ ++ if (sps->delta_pic_order_always_zero_flag) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO; ++ ++ if (sps->gaps_in_frame_num_allowed_flag) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED; ++ ++ if (sps->frame_mbs_only_flag) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY; ++ ++ if (sps->mb_aff) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; ++ ++ if (sps->direct_8x8_inference_flag) ++ ctrl->flags |= V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE; ++} ++ ++static void fill_pps(struct v4l2_ctrl_h264_pps *ctrl, const H264Context *h) ++{ ++ const SPS *sps = h->ps.sps; ++ const PPS *pps = h->ps.pps; ++ const H264SliceContext *sl = &h->slice_ctx[0]; ++ int qp_bd_offset = 6 * (sps->bit_depth_luma - 8); ++ ++ *ctrl = (struct v4l2_ctrl_h264_pps) { ++ .pic_parameter_set_id = sl->pps_id, ++ .seq_parameter_set_id = pps->sps_id, ++ .num_slice_groups_minus1 = pps->slice_group_count - 1, ++ .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1, ++ .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1, ++ .weighted_bipred_idc = pps->weighted_bipred_idc, ++ .pic_init_qp_minus26 = pps->init_qp - 26 - qp_bd_offset, ++ .pic_init_qs_minus26 = pps->init_qs - 26 - qp_bd_offset, ++ .chroma_qp_index_offset = pps->chroma_qp_index_offset[0], ++ .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1], ++ }; ++ ++ if (pps->cabac) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; ++ ++ if (pps->pic_order_present) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT; ++ ++ if (pps->weighted_pred) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; ++ ++ if (pps->deblocking_filter_parameters_present) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; ++ ++ if (pps->constrained_intra_pred) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED; ++ ++ if (pps->redundant_pic_cnt_present) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; ++ ++ if (pps->transform_8x8_mode) ++ ctrl->flags |= V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE; ++ ++ /* FFmpeg always provide a scaling matrix */ ++ ctrl->flags |= V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT; ++} ++ ++static int v4l2_request_h264_start_frame(AVCodecContext *avctx, ++ av_unused const uint8_t *buffer, ++ av_unused uint32_t size) ++{ ++ const H264Context *h = avctx->priv_data; ++ const PPS *pps = h->ps.pps; ++ const SPS *sps = h->ps.sps; ++ const H264SliceContext *sl = &h->slice_ctx[0]; ++ V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; ++ int ret; ++ ++ ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_pic_ptr->f); ++ if (ret) ++ return ret; ++ ++ fill_sps(&controls->sps, h); ++ fill_pps(&controls->pps, h); ++ ++ memcpy(controls->scaling_matrix.scaling_list_4x4, pps->scaling_matrix4, ++ sizeof(controls->scaling_matrix.scaling_list_4x4)); ++ memcpy(controls->scaling_matrix.scaling_list_8x8[0], pps->scaling_matrix8[0], ++ sizeof(controls->scaling_matrix.scaling_list_8x8[0])); ++ memcpy(controls->scaling_matrix.scaling_list_8x8[1], pps->scaling_matrix8[3], ++ sizeof(controls->scaling_matrix.scaling_list_8x8[1])); ++ ++ if (sps->chroma_format_idc == 3) { ++ memcpy(controls->scaling_matrix.scaling_list_8x8[2], pps->scaling_matrix8[1], ++ sizeof(controls->scaling_matrix.scaling_list_8x8[2])); ++ memcpy(controls->scaling_matrix.scaling_list_8x8[3], pps->scaling_matrix8[4], ++ sizeof(controls->scaling_matrix.scaling_list_8x8[3])); ++ memcpy(controls->scaling_matrix.scaling_list_8x8[4], pps->scaling_matrix8[2], ++ sizeof(controls->scaling_matrix.scaling_list_8x8[4])); ++ memcpy(controls->scaling_matrix.scaling_list_8x8[5], pps->scaling_matrix8[5], ++ sizeof(controls->scaling_matrix.scaling_list_8x8[5])); ++ } ++ ++ controls->decode_params = (struct v4l2_ctrl_h264_decode_params) { ++ .nal_ref_idc = h->nal_ref_idc, ++ .frame_num = h->poc.frame_num, ++ .top_field_order_cnt = h->cur_pic_ptr->field_poc[0] != INT_MAX ? ++ h->cur_pic_ptr->field_poc[0] : 0, ++ .bottom_field_order_cnt = h->cur_pic_ptr->field_poc[1] != INT_MAX ? ++ h->cur_pic_ptr->field_poc[1] : 0, ++ .idr_pic_id = sl->idr_pic_id, ++ .pic_order_cnt_lsb = sl->poc_lsb, ++ .delta_pic_order_cnt_bottom = sl->delta_poc_bottom, ++ .delta_pic_order_cnt0 = sl->delta_poc[0], ++ .delta_pic_order_cnt1 = sl->delta_poc[1], ++ /* Size in bits of dec_ref_pic_marking() syntax element. */ ++ .dec_ref_pic_marking_bit_size = sl->ref_pic_marking_bit_size, ++ /* Size in bits of pic order count syntax. */ ++ .pic_order_cnt_bit_size = sl->pic_order_cnt_bit_size, ++ .slice_group_change_cycle = 0, /* slice group not supported by FFmpeg */ ++ }; ++ ++ if (h->picture_idr) ++ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; ++ ++ if (FIELD_PICTURE(h)) ++ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC; ++ ++ if (h->picture_structure == PICT_BOTTOM_FIELD) ++ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD; ++ ++#if defined(V4L2_H264_DECODE_PARAM_FLAG_PFRAME) ++ if (sl->slice_type_nos == AV_PICTURE_TYPE_P) ++ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_PFRAME; ++#endif ++ ++#if defined(V4L2_H264_DECODE_PARAM_FLAG_BFRAME) ++ if (sl->slice_type_nos == AV_PICTURE_TYPE_B) ++ controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BFRAME; ++#endif ++ ++ fill_dpb(&controls->decode_params, h); ++ ++ controls->first_slice = true; ++ controls->num_slices = 0; ++ ++ return 0; ++} ++ ++static int v4l2_request_h264_queue_decode(AVCodecContext *avctx, bool last_slice) ++{ ++ const H264Context *h = avctx->priv_data; ++ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; ++ V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; ++ ++ struct v4l2_ext_control control[] = { ++ { ++ .id = V4L2_CID_STATELESS_H264_SPS, ++ .ptr = &controls->sps, ++ .size = sizeof(controls->sps), ++ }, ++ { ++ .id = V4L2_CID_STATELESS_H264_PPS, ++ .ptr = &controls->pps, ++ .size = sizeof(controls->pps), ++ }, ++ { ++ .id = V4L2_CID_STATELESS_H264_SCALING_MATRIX, ++ .ptr = &controls->scaling_matrix, ++ .size = sizeof(controls->scaling_matrix), ++ }, ++ { ++ .id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, ++ .ptr = &controls->decode_params, ++ .size = sizeof(controls->decode_params), ++ }, ++ { ++ .id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, ++ .ptr = &controls->slice_params, ++ .size = sizeof(controls->slice_params), ++ }, ++ { ++ .id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS, ++ .ptr = &controls->pred_weights, ++ .size = sizeof(controls->pred_weights), ++ }, ++ }; ++ ++ if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) { ++ int count = FF_ARRAY_ELEMS(control) - (controls->pred_weights_required ? 0 : 1); ++ return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count, ++ controls->first_slice, last_slice); ++ } ++ ++ return ff_v4l2_request_decode_frame(avctx, &controls->pic, ++ control, FF_ARRAY_ELEMS(control) - 2); ++} ++ ++static int v4l2_request_h264_decode_slice(AVCodecContext *avctx, ++ const uint8_t *buffer, uint32_t size) ++{ ++ const H264Context *h = avctx->priv_data; ++ const PPS *pps = h->ps.pps; ++ const H264SliceContext *sl = &h->slice_ctx[0]; ++ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; ++ V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; ++ int i, ret, count; ++ ++ if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && ++ controls->num_slices) { ++ ret = v4l2_request_h264_queue_decode(avctx, false); ++ if (ret) ++ return ret; ++ ++ ff_v4l2_request_reset_picture(avctx, &controls->pic); ++ controls->first_slice = 0; ++ } ++ ++ if (ctx->start_code == V4L2_STATELESS_H264_START_CODE_ANNEX_B) { ++ ret = ff_v4l2_request_append_output(avctx, &controls->pic, ++ nalu_slice_start_code, 3); ++ if (ret) ++ return ret; ++ } ++ ++ ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); ++ if (ret) ++ return ret; ++ ++ if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) ++ return 0; ++ ++ controls->slice_params = (struct v4l2_ctrl_h264_slice_params) { ++ /* Offset in bits to slice_data() from the beginning of this slice. */ ++ .header_bit_size = get_bits_count(&sl->gb), ++ ++ .first_mb_in_slice = sl->first_mb_addr, ++ ++ .slice_type = ff_h264_get_slice_type(sl), ++ .colour_plane_id = 0, /* separate colour plane not supported by FFmpeg */ ++ .redundant_pic_cnt = sl->redundant_pic_count, ++ .cabac_init_idc = sl->cabac_init_idc, ++ .slice_qp_delta = sl->qscale - pps->init_qp, ++ .slice_qs_delta = 0, /* not implemented by FFmpeg */ ++ .disable_deblocking_filter_idc = sl->deblocking_filter < 2 ? ++ !sl->deblocking_filter : ++ sl->deblocking_filter, ++ .slice_alpha_c0_offset_div2 = sl->slice_alpha_c0_offset / 2, ++ .slice_beta_offset_div2 = sl->slice_beta_offset / 2, ++ .num_ref_idx_l0_active_minus1 = sl->list_count > 0 ? sl->ref_count[0] - 1 : 0, ++ .num_ref_idx_l1_active_minus1 = sl->list_count > 1 ? sl->ref_count[1] - 1 : 0, ++ }; ++ ++ if (sl->slice_type == AV_PICTURE_TYPE_B && sl->direct_spatial_mv_pred) ++ controls->slice_params.flags |= V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; ++ ++ /* V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH: not implemented by FFmpeg */ ++ ++ controls->pred_weights_required = ++ V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(&controls->pps, &controls->slice_params); ++ if (controls->pred_weights_required) { ++ controls->pred_weights.chroma_log2_weight_denom = sl->pwt.chroma_log2_weight_denom; ++ controls->pred_weights.luma_log2_weight_denom = sl->pwt.luma_log2_weight_denom; ++ } ++ ++ count = sl->list_count > 0 ? sl->ref_count[0] : 0; ++ for (i = 0; i < count; i++) ++ fill_ref_list(&controls->slice_params.ref_pic_list0[i], ++ &controls->decode_params, &sl->ref_list[0][i]); ++ if (count && controls->pred_weights_required) ++ fill_weight_factors(&controls->pred_weights.weight_factors[0], 0, sl); ++ ++ count = sl->list_count > 1 ? sl->ref_count[1] : 0; ++ for (i = 0; i < count; i++) ++ fill_ref_list(&controls->slice_params.ref_pic_list1[i], ++ &controls->decode_params, &sl->ref_list[1][i]); ++ if (count && controls->pred_weights_required) ++ fill_weight_factors(&controls->pred_weights.weight_factors[1], 1, sl); ++ ++ controls->num_slices++; ++ return 0; ++} ++ ++static int v4l2_request_h264_end_frame(AVCodecContext *avctx) ++{ ++ return v4l2_request_h264_queue_decode(avctx, true); ++} ++ ++static int v4l2_request_h264_post_probe(AVCodecContext *avctx) ++{ ++ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; ++ ++ struct v4l2_ext_control control[] = { ++ { .id = V4L2_CID_STATELESS_H264_DECODE_MODE, }, ++ { .id = V4L2_CID_STATELESS_H264_START_CODE, }, ++ }; ++ ++ ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, ++ V4L2_CID_STATELESS_H264_DECODE_MODE); ++ if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && ++ ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED) { ++ av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n", ++ ctx->decode_mode); ++ return AVERROR(EINVAL); ++ } ++ ++ ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, ++ V4L2_CID_STATELESS_H264_START_CODE); ++ if (ctx->start_code != V4L2_STATELESS_H264_START_CODE_NONE && ++ ctx->start_code != V4L2_STATELESS_H264_START_CODE_ANNEX_B) { ++ av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n", ++ ctx->start_code); ++ return AVERROR(EINVAL); ++ } ++ ++ // TODO: check V4L2_CID_MPEG_VIDEO_H264_PROFILE control ++ // TODO: check V4L2_CID_MPEG_VIDEO_H264_LEVEL control ++ ++ control[0].value = ctx->decode_mode; ++ control[1].value = ctx->start_code; ++ ++ return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control)); ++} ++ ++static int v4l2_request_h264_init(AVCodecContext *avctx) ++{ ++ V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; ++ const H264Context *h = avctx->priv_data; ++ struct v4l2_ctrl_h264_sps sps; ++ ++ struct v4l2_ext_control control[] = { ++ { ++ .id = V4L2_CID_STATELESS_H264_SPS, ++ .ptr = &sps, ++ .size = sizeof(sps), ++ }, ++ }; ++ ++ fill_sps(&sps, h); ++ ++ ctx->base.post_probe = v4l2_request_h264_post_probe; ++ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_H264_SLICE, ++ 4 * 1024 * 1024, ++ control, FF_ARRAY_ELEMS(control)); ++} ++ ++const FFHWAccel ff_h264_v4l2request_hwaccel = { ++ .p.name = "h264_v4l2request", ++ .p.type = AVMEDIA_TYPE_VIDEO, ++ .p.id = AV_CODEC_ID_H264, ++ .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, ++ .start_frame = v4l2_request_h264_start_frame, ++ .decode_slice = v4l2_request_h264_decode_slice, ++ .end_frame = v4l2_request_h264_end_frame, ++ .flush = ff_v4l2_request_flush, ++ .frame_priv_data_size = sizeof(V4L2RequestControlsH264), ++ .init = v4l2_request_h264_init, ++ .uninit = ff_v4l2_request_uninit, ++ .priv_data_size = sizeof(V4L2RequestContextH264), ++ .frame_params = ff_v4l2_request_frame_params, +}; -diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c -index db2419deaf70..ad5e6e8f2b1e 100644 ---- a/libavcodec/vp8.c -+++ b/libavcodec/vp8.c -@@ -206,6 +206,9 @@ static enum AVPixelFormat get_pixel_format(VP8Context *s) - #endif - #if CONFIG_VP8_NVDEC_HWACCEL - AV_PIX_FMT_CUDA, -+#endif -+#if CONFIG_VP8_V4L2REQUEST_HWACCEL -+ AV_PIX_FMT_DRM_PRIME, - #endif - AV_PIX_FMT_YUV420P, - AV_PIX_FMT_NONE, -@@ -3007,6 +3010,9 @@ const FFCodec ff_vp8_decoder = { - #endif - #if CONFIG_VP8_NVDEC_HWACCEL - HWACCEL_NVDEC(vp8), -+#endif -+#if CONFIG_VP8_V4L2REQUEST_HWACCEL -+ HWACCEL_V4L2REQUEST(vp8), - #endif - NULL - }, -From ec4f9a58dd3044a74f3f05bae8901d47e2f3e49f Mon Sep 17 00:00:00 2001 -From: Jernej Skrabec -Date: Sat, 15 Dec 2018 22:32:16 +0100 -Subject: [PATCH 07/11] Add V4L2 request API hevc hwaccel +From 0b3c85ab94619e449026145a9795f2013732977c Mon Sep 17 00:00:00 2001 +From: Jernej Skrabec +Date: Tue, 6 Aug 2024 09:06:07 +0000 +Subject: [PATCH 08/11] avcodec: Add V4L2 Request API hevc hwaccel -Signed-off-by: Jernej Skrabec -Signed-off-by: Jonas Karlman +Add a V4L2 Request API hwaccel for HEVC, supporting both slice and +frame decoding modes. + +Support for HEVC is enabled when Linux kernel headers declare the +control id V4L2_CID_STATELESS_HEVC_SPS, added in v6.0. + +Co-developed-by: Benjamin Gaignard Signed-off-by: Benjamin Gaignard +Co-developed-by: Alex Bee Signed-off-by: Alex Bee +Signed-off-by: Jernej Skrabec +Co-developed-by: Jonas Karlman +Signed-off-by: Jonas Karlman --- - configure | 3 + + configure | 5 + libavcodec/Makefile | 1 + - libavcodec/hevcdec.c | 10 + + libavcodec/hevc/hevcdec.c | 10 + libavcodec/hwaccels.h | 1 + - libavcodec/v4l2_request_hevc.c | 681 +++++++++++++++++++++++++++++++++ - 5 files changed, 696 insertions(+) + libavcodec/v4l2_request_hevc.c | 745 +++++++++++++++++++++++++++++++++ + 5 files changed, 762 insertions(+) create mode 100644 libavcodec/v4l2_request_hevc.c diff --git a/configure b/configure -index d6fd3281a38c..e6a4f0013218 100755 +index 6866a61153..7a2901d960 100755 --- a/configure +++ b/configure -@@ -3049,6 +3049,8 @@ hevc_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_HEVC" +@@ -2491,6 +2491,7 @@ TYPES_LIST=" + struct_sockaddr_sa_len + struct_sockaddr_storage + struct_stat_st_mtim_tv_nsec ++ struct_v4l2_ctrl_hevc_decode_params_num_delta_pocs_of_ref_rps_idx + struct_v4l2_frmivalenum_discrete + struct_mfxConfigInterface + " +@@ -3206,6 +3207,8 @@ hevc_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_HEVC" hevc_dxva2_hwaccel_select="hevc_decoder" hevc_nvdec_hwaccel_deps="nvdec" hevc_nvdec_hwaccel_select="hevc_decoder" @@ -2502,49 +3559,54 @@ index d6fd3281a38c..e6a4f0013218 100755 hevc_vaapi_hwaccel_deps="vaapi VAPictureParameterBufferHEVC" hevc_vaapi_hwaccel_select="hevc_decoder" hevc_vdpau_hwaccel_deps="vdpau VdpPictureInfoHEVC" -@@ -6894,6 +6896,7 @@ fi +@@ -7207,10 +7210,12 @@ fi - check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns - check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_H264_SLICE;" -+check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_HEVC_SLICE;" - check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_MPEG2_SLICE;" - check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_VP8_FRAME;" + if enabled v4l2_request; then + check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_H264_DECODE_MODE" ++ check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS" + check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" + check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns + check_pkg_config libudev libudev libudev.h udev_new ++ check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx + fi + check_headers sys/videoio.h diff --git a/libavcodec/Makefile b/libavcodec/Makefile -index 7da4fd1a8718..cd08740e75f1 100644 +index 00bc1cf8f2..fa78523b76 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile -@@ -998,6 +998,7 @@ OBJS-$(CONFIG_HEVC_D3D11VA_HWACCEL) += dxva2_hevc.o - OBJS-$(CONFIG_HEVC_DXVA2_HWACCEL) += dxva2_hevc.o +@@ -1027,6 +1027,7 @@ OBJS-$(CONFIG_HEVC_DXVA2_HWACCEL) += dxva2_hevc.o + OBJS-$(CONFIG_HEVC_D3D12VA_HWACCEL) += dxva2_hevc.o d3d12va_hevc.o OBJS-$(CONFIG_HEVC_NVDEC_HWACCEL) += nvdec_hevc.o OBJS-$(CONFIG_HEVC_QSV_HWACCEL) += qsvdec.o +OBJS-$(CONFIG_HEVC_V4L2REQUEST_HWACCEL) += v4l2_request_hevc.o OBJS-$(CONFIG_HEVC_VAAPI_HWACCEL) += vaapi_hevc.o h265_profile_level.o OBJS-$(CONFIG_HEVC_VDPAU_HWACCEL) += vdpau_hevc.o h265_profile_level.o - OBJS-$(CONFIG_MJPEG_NVDEC_HWACCEL) += nvdec_mjpeg.o -diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c -index 0e2844f47cef..6791dc9554a6 100644 ---- a/libavcodec/hevcdec.c -+++ b/libavcodec/hevcdec.c -@@ -403,6 +403,7 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) - #define HWACCEL_MAX (CONFIG_HEVC_DXVA2_HWACCEL + \ + OBJS-$(CONFIG_HEVC_VULKAN_HWACCEL) += vulkan_decode.o vulkan_hevc.o +diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c +index 0dc24f82f8..4b14a931a2 100644 +--- a/libavcodec/hevc/hevcdec.c ++++ b/libavcodec/hevc/hevcdec.c +@@ -527,6 +527,7 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) CONFIG_HEVC_D3D11VA_HWACCEL * 2 + \ + CONFIG_HEVC_D3D12VA_HWACCEL + \ CONFIG_HEVC_NVDEC_HWACCEL + \ + CONFIG_HEVC_V4L2REQUEST_HWACCEL + \ CONFIG_HEVC_VAAPI_HWACCEL + \ CONFIG_HEVC_VIDEOTOOLBOX_HWACCEL + \ - CONFIG_HEVC_VDPAU_HWACCEL) -@@ -429,6 +430,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) + CONFIG_HEVC_VDPAU_HWACCEL + \ +@@ -561,6 +562,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) #endif - #if CONFIG_HEVC_VIDEOTOOLBOX_HWACCEL - *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX; + #if CONFIG_HEVC_VULKAN_HWACCEL + *fmt++ = AV_PIX_FMT_VULKAN; +#endif +#if CONFIG_HEVC_V4L2REQUEST_HWACCEL + *fmt++ = AV_PIX_FMT_DRM_PRIME; #endif break; case AV_PIX_FMT_YUV420P10: -@@ -450,6 +454,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) +@@ -588,6 +592,9 @@ static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps) #endif #if CONFIG_HEVC_NVDEC_HWACCEL *fmt++ = AV_PIX_FMT_CUDA; @@ -2554,10 +3616,10 @@ index 0e2844f47cef..6791dc9554a6 100644 #endif break; case AV_PIX_FMT_YUV444P: -@@ -3742,6 +3749,9 @@ const FFCodec ff_hevc_decoder = { +@@ -4107,6 +4114,9 @@ const FFCodec ff_hevc_decoder = { #endif - #if CONFIG_HEVC_VIDEOTOOLBOX_HWACCEL - HWACCEL_VIDEOTOOLBOX(hevc), + #if CONFIG_HEVC_VULKAN_HWACCEL + HWACCEL_VULKAN(hevc), +#endif +#if CONFIG_HEVC_V4L2REQUEST_HWACCEL + HWACCEL_V4L2REQUEST(hevc), @@ -2565,23 +3627,23 @@ index 0e2844f47cef..6791dc9554a6 100644 NULL }, diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h -index 6f9f0780019b..e4e4abc0606a 100644 +index 024e021d73..ad145fc07c 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h -@@ -41,6 +41,7 @@ extern const AVHWAccel ff_hevc_d3d11va_hwaccel; - extern const AVHWAccel ff_hevc_d3d11va2_hwaccel; - extern const AVHWAccel ff_hevc_dxva2_hwaccel; - extern const AVHWAccel ff_hevc_nvdec_hwaccel; -+extern const AVHWAccel ff_hevc_v4l2request_hwaccel; - extern const AVHWAccel ff_hevc_vaapi_hwaccel; - extern const AVHWAccel ff_hevc_vdpau_hwaccel; - extern const AVHWAccel ff_hevc_videotoolbox_hwaccel; +@@ -44,6 +44,7 @@ extern const struct FFHWAccel ff_hevc_d3d11va2_hwaccel; + extern const struct FFHWAccel ff_hevc_d3d12va_hwaccel; + extern const struct FFHWAccel ff_hevc_dxva2_hwaccel; + extern const struct FFHWAccel ff_hevc_nvdec_hwaccel; ++extern const struct FFHWAccel ff_hevc_v4l2request_hwaccel; + extern const struct FFHWAccel ff_hevc_vaapi_hwaccel; + extern const struct FFHWAccel ff_hevc_vdpau_hwaccel; + extern const struct FFHWAccel ff_hevc_videotoolbox_hwaccel; diff --git a/libavcodec/v4l2_request_hevc.c b/libavcodec/v4l2_request_hevc.c new file mode 100644 -index 000000000000..57a0d5e78092 +index 0000000000..9fabe468b7 --- /dev/null +++ b/libavcodec/v4l2_request_hevc.c -@@ -0,0 +1,681 @@ +@@ -0,0 +1,745 @@ +/* + * This file is part of FFmpeg. + * @@ -2600,52 +3662,60 @@ index 000000000000..57a0d5e78092 + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + -+#include "hevcdec.h" ++#include "config.h" ++ ++#include "libavutil/mem.h" ++#include "hevc/hevcdec.h" ++#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "internal.h" +#include "v4l2_request.h" + -+#define MAX_SLICES 600 // as per HEVC spec ? +#define V4L2_HEVC_CONTROLS_MAX 6 + -+typedef struct V4L2RequestControlsHEVC { -+ struct v4l2_ctrl_hevc_sps sps; -+ struct v4l2_ctrl_hevc_pps pps; -+ struct v4l2_ctrl_hevc_decode_params dec_params; -+ struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix; -+ struct v4l2_ctrl_hevc_slice_params slice_params[MAX_SLICES]; -+ __u32 *entry_point_offsets; -+ unsigned int num_entry_point_offsets; -+ int first_slice; -+ int num_slices; -+} V4L2RequestControlsHEVC; -+ +typedef struct V4L2RequestContextHEVC { + V4L2RequestContext base; -+ unsigned int decode_mode; -+ unsigned int start_code; -+ __u32 max_slices; -+ unsigned int supports_entry_point_offsets; -+ unsigned int supports_slices; -+ unsigned int supports_scaling_matrix; ++ enum v4l2_stateless_hevc_decode_mode decode_mode; ++ enum v4l2_stateless_hevc_start_code start_code; ++ unsigned int max_slice_params; ++ unsigned int max_entry_point_offsets; ++ bool has_scaling_matrix; +} V4L2RequestContextHEVC; + ++typedef struct V4L2RequestControlsHEVC { ++ V4L2RequestPictureContext pic; ++ struct v4l2_ctrl_hevc_sps sps; ++ struct v4l2_ctrl_hevc_pps pps; ++ struct v4l2_ctrl_hevc_decode_params decode_params; ++ struct v4l2_ctrl_hevc_scaling_matrix scaling_matrix; ++ struct v4l2_ctrl_hevc_slice_params slice_params; ++ struct v4l2_ctrl_hevc_slice_params *frame_slice_params; ++ unsigned int allocated_slice_params; ++ unsigned int num_slice_params; ++ uint32_t *entry_point_offsets; ++ unsigned int allocated_entry_point_offsets; ++ unsigned int num_entry_point_offsets; ++ bool first_slice; ++} V4L2RequestControlsHEVC; ++ +static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 }; + -+static void v4l2_request_hevc_fill_pred_table(const HEVCContext *h, struct v4l2_hevc_pred_weight_table *table) ++static void fill_pred_weight_table(struct v4l2_hevc_pred_weight_table *table, ++ const HEVCContext *h) +{ + int32_t luma_weight_denom, chroma_weight_denom; + const SliceHeader *sh = &h->sh; + + if (sh->slice_type == HEVC_SLICE_I || -+ (sh->slice_type == HEVC_SLICE_P && !h->ps.pps->weighted_pred_flag) || -+ (sh->slice_type == HEVC_SLICE_B && !h->ps.pps->weighted_bipred_flag)) ++ (sh->slice_type == HEVC_SLICE_P && !h->pps->weighted_pred_flag) || ++ (sh->slice_type == HEVC_SLICE_B && !h->pps->weighted_bipred_flag)) + return; + + table->luma_log2_weight_denom = sh->luma_log2_weight_denom; + -+ if (h->ps.sps->chroma_format_idc) -+ table->delta_chroma_log2_weight_denom = sh->chroma_log2_weight_denom - sh->luma_log2_weight_denom; ++ if (h->pps->sps->chroma_format_idc) ++ table->delta_chroma_log2_weight_denom = sh->chroma_log2_weight_denom - ++ sh->luma_log2_weight_denom; + + luma_weight_denom = (1 << sh->luma_log2_weight_denom); + chroma_weight_denom = (1 << sh->chroma_log2_weight_denom); @@ -2673,17 +3743,17 @@ index 000000000000..57a0d5e78092 +} + +static uint8_t get_ref_pic_index(const HEVCContext *h, const HEVCFrame *frame, -+ struct v4l2_ctrl_hevc_decode_params *dec_params) ++ struct v4l2_ctrl_hevc_decode_params *decode_params) +{ + uint64_t timestamp; + -+ if (!frame) ++ if (!frame || !frame->f) + return 0; + -+ timestamp = ff_v4l2_request_get_capture_timestamp(frame->frame); ++ timestamp = ff_v4l2_request_get_capture_timestamp(frame->f); + -+ for (uint8_t i = 0; i < dec_params->num_active_dpb_entries; i++) { -+ struct v4l2_hevc_dpb_entry *entry = &dec_params->dpb[i]; ++ for (uint8_t i = 0; i < decode_params->num_active_dpb_entries; i++) { ++ struct v4l2_hevc_dpb_entry *entry = &decode_params->dpb[i]; + if (entry->timestamp == timestamp) + return i; + } @@ -2691,14 +3761,16 @@ index 000000000000..57a0d5e78092 + return 0; +} + -+static void fill_dec_params(struct v4l2_ctrl_hevc_decode_params *dec_params, const HEVCContext *h) ++static void fill_decode_params(struct v4l2_ctrl_hevc_decode_params *decode_params, ++ const HEVCContext *h) +{ -+ const HEVCFrame *pic = h->ref; ++ const HEVCFrame *pic = h->cur_frame; ++ const HEVCLayerContext *l = &h->layers[h->cur_layer]; + const SliceHeader *sh = &h->sh; + int i, entries = 0; + -+ *dec_params = (struct v4l2_ctrl_hevc_decode_params) { -+ .pic_order_cnt_val = pic->poc, /* FIXME: is this same as slice_params->slice_pic_order_cnt ? */ ++ *decode_params = (struct v4l2_ctrl_hevc_decode_params) { ++ .pic_order_cnt_val = h->poc, + .short_term_ref_pic_set_size = sh->short_term_ref_pic_set_size, + .long_term_ref_pic_set_size = sh->long_term_ref_pic_set_size, + .num_poc_st_curr_before = h->rps[ST_CURR_BEF].nb_refs, @@ -2706,13 +3778,20 @@ index 000000000000..57a0d5e78092 + .num_poc_lt_curr = h->rps[LT_CURR].nb_refs, + }; + -+ for (i = 0; i < FF_ARRAY_ELEMS(h->DPB); i++) { -+ const HEVCFrame *frame = &h->DPB[i]; -+ if (frame != pic && (frame->flags & (HEVC_FRAME_FLAG_LONG_REF | HEVC_FRAME_FLAG_SHORT_REF))) { -+ struct v4l2_hevc_dpb_entry *entry = &dec_params->dpb[entries++]; ++#if HAVE_STRUCT_V4L2_CTRL_HEVC_DECODE_PARAMS_NUM_DELTA_POCS_OF_REF_RPS_IDX ++ if (h->sh.short_term_ref_pic_set_sps_flag == 0 && h->sh.short_term_rps) ++ decode_params->num_delta_pocs_of_ref_rps_idx = ++ h->sh.short_term_rps->rps_idx_num_delta_pocs; ++#endif + -+ entry->timestamp = ff_v4l2_request_get_capture_timestamp(frame->frame); -+ entry->field_pic = frame->frame->interlaced_frame; ++ for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++) { ++ const HEVCFrame *frame = &l->DPB[i]; ++ if (frame != pic && ++ (frame->flags & (HEVC_FRAME_FLAG_LONG_REF | HEVC_FRAME_FLAG_SHORT_REF))) { ++ struct v4l2_hevc_dpb_entry *entry = &decode_params->dpb[entries++]; ++ ++ entry->timestamp = ff_v4l2_request_get_capture_timestamp(frame->f); ++ entry->field_pic = !!(frame->f->flags & AV_FRAME_FLAG_INTERLACED); + entry->flags = 0; + if (frame->flags & HEVC_FRAME_FLAG_LONG_REF) + entry->flags |= V4L2_HEVC_DPB_ENTRY_LONG_TERM_REFERENCE; @@ -2721,38 +3800,39 @@ index 000000000000..57a0d5e78092 + } + } + -+ dec_params->num_active_dpb_entries = entries; ++ decode_params->num_active_dpb_entries = entries; + + if (IS_IRAP(h)) -+ dec_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC; ++ decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC; + + if (IS_IDR(h)) -+ dec_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC; ++ decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC; + + if (sh->no_output_of_prior_pics_flag) -+ dec_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_NO_OUTPUT_OF_PRIOR; ++ decode_params->flags |= V4L2_HEVC_DECODE_PARAM_FLAG_NO_OUTPUT_OF_PRIOR; + + for (i = 0; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { -+ dec_params->poc_st_curr_before[i] = get_ref_pic_index(h, h->rps[ST_CURR_BEF].ref[i], dec_params); -+ dec_params->poc_st_curr_after[i] = get_ref_pic_index(h, h->rps[ST_CURR_AFT].ref[i], dec_params); -+ dec_params->poc_lt_curr[i] = get_ref_pic_index(h, h->rps[LT_CURR].ref[i], dec_params); ++ decode_params->poc_st_curr_before[i] = ++ get_ref_pic_index(h, h->rps[ST_CURR_BEF].ref[i], decode_params); ++ decode_params->poc_st_curr_after[i] = ++ get_ref_pic_index(h, h->rps[ST_CURR_AFT].ref[i], decode_params); ++ decode_params->poc_lt_curr[i] = ++ get_ref_pic_index(h, h->rps[LT_CURR].ref[i], decode_params); + } +} + -+static int v4l2_request_hevc_fill_slice_params(const HEVCContext *h, -+ V4L2RequestControlsHEVC *controls, -+ int slice) ++static int fill_slice_params(V4L2RequestControlsHEVC *controls, int slice, ++ bool max_entry_point_offsets, const HEVCContext *h) +{ -+ struct v4l2_ctrl_hevc_slice_params *slice_params = &controls->slice_params[slice]; -+ struct v4l2_ctrl_hevc_decode_params *dec_params = &controls->dec_params; -+ const HEVCFrame *pic = h->ref; ++ struct v4l2_ctrl_hevc_slice_params *slice_params = &controls->frame_slice_params[slice]; ++ struct v4l2_ctrl_hevc_decode_params *decode_params = &controls->decode_params; + const SliceHeader *sh = &h->sh; + RefPicList *rpl; -+ int i; ++ int i, offsets; + + *slice_params = (struct v4l2_ctrl_hevc_slice_params) { + .bit_size = 0, -+ .data_byte_offset = (get_bits_count(&h->HEVClc->gb) + 1 + 7) / 8, ++ .data_byte_offset = controls->pic.output->used + sh->data_offset, + .num_entry_point_offsets = sh->num_entry_point_offsets, + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: NAL unit header */ @@ -2762,11 +3842,13 @@ index 000000000000..57a0d5e78092 + /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ + .slice_type = sh->slice_type, + .colour_plane_id = sh->colour_plane_id, -+ .slice_pic_order_cnt = pic->poc, ++ .slice_pic_order_cnt = sh->poc, + .num_ref_idx_l0_active_minus1 = sh->nb_refs[L0] ? sh->nb_refs[L0] - 1 : 0, + .num_ref_idx_l1_active_minus1 = sh->nb_refs[L1] ? sh->nb_refs[L1] - 1 : 0, -+ .collocated_ref_idx = sh->slice_temporal_mvp_enabled_flag ? sh->collocated_ref_idx : 0, -+ .five_minus_max_num_merge_cand = sh->slice_type == HEVC_SLICE_I ? 0 : 5 - sh->max_num_merge_cand, ++ .collocated_ref_idx = sh->slice_temporal_mvp_enabled_flag ? ++ sh->collocated_ref_idx : 0, ++ .five_minus_max_num_merge_cand = sh->slice_type == HEVC_SLICE_I ? ++ 0 : 5 - sh->max_num_merge_cand, + .slice_qp_delta = sh->slice_qp_delta, + .slice_cb_qp_offset = sh->slice_cb_qp_offset, + .slice_cr_qp_offset = sh->slice_cr_qp_offset, @@ -2785,6 +3867,12 @@ index 000000000000..57a0d5e78092 + .long_term_ref_pic_set_size = sh->long_term_ref_pic_set_size, + }; + ++ if (h->pps->pps_slice_act_qp_offsets_present_flag) { ++ slice_params->slice_act_y_qp_offset = sh->slice_act_y_qp_offset; ++ slice_params->slice_act_cb_qp_offset = sh->slice_act_cb_qp_offset; ++ slice_params->slice_act_cr_qp_offset = sh->slice_act_cr_qp_offset; ++ } ++ + if (sh->slice_sample_adaptive_offset_flag[0]) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_SAO_LUMA; + @@ -2803,7 +3891,8 @@ index 000000000000..57a0d5e78092 + if (sh->collocated_list == L0) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_COLLOCATED_FROM_L0; + -+ /* TODO: V4L2_HEVC_SLICE_PARAMS_FLAG_USE_INTEGER_MV */ ++ if (sh->use_integer_mv_flag) ++ slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_USE_INTEGER_MV; + + if (sh->disable_deblocking_filter_flag) + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_DEBLOCKING_FILTER_DISABLED; @@ -2815,37 +3904,45 @@ index 000000000000..57a0d5e78092 + slice_params->flags |= V4L2_HEVC_SLICE_PARAMS_FLAG_DEPENDENT_SLICE_SEGMENT; + + if (sh->slice_type != HEVC_SLICE_I) { -+ rpl = &h->ref->refPicList[0]; ++ rpl = &h->cur_frame->refPicList[0]; + for (i = 0; i < rpl->nb_refs; i++) -+ slice_params->ref_idx_l0[i] = get_ref_pic_index(h, rpl->ref[i], dec_params); ++ slice_params->ref_idx_l0[i] = get_ref_pic_index(h, rpl->ref[i], decode_params); + } + + if (sh->slice_type == HEVC_SLICE_B) { -+ rpl = &h->ref->refPicList[1]; ++ rpl = &h->cur_frame->refPicList[1]; + for (i = 0; i < rpl->nb_refs; i++) -+ slice_params->ref_idx_l1[i] = get_ref_pic_index(h, rpl->ref[i], dec_params); ++ slice_params->ref_idx_l1[i] = get_ref_pic_index(h, rpl->ref[i], decode_params); + } + -+ v4l2_request_hevc_fill_pred_table(h, &slice_params->pred_weight_table); ++ fill_pred_weight_table(&slice_params->pred_weight_table, h); + -+ if (controls->num_entry_point_offsets < sh->num_entry_point_offsets) { -+ av_freep(&controls->entry_point_offsets); -+ controls->entry_point_offsets = av_mallocz(sizeof(*controls->entry_point_offsets) * sh->num_entry_point_offsets); -+ if (!controls->entry_point_offsets) ++ if (!max_entry_point_offsets) ++ return 0; ++ ++ if (controls->allocated_entry_point_offsets < controls->num_entry_point_offsets + sh->num_entry_point_offsets) { ++ void *entry_point_offsets = controls->entry_point_offsets; ++ offsets = controls->allocated_entry_point_offsets == 0 ? 128 : controls->allocated_entry_point_offsets * 2; ++ while (controls->num_entry_point_offsets + sh->num_entry_point_offsets > offsets) ++ offsets *= 2; ++ entry_point_offsets = av_realloc_array(entry_point_offsets, offsets, sizeof(*controls->entry_point_offsets)); ++ if (!entry_point_offsets) + return AVERROR(ENOMEM); -+ controls->num_entry_point_offsets = sh->num_entry_point_offsets; ++ controls->entry_point_offsets = entry_point_offsets; ++ controls->allocated_entry_point_offsets = offsets; + } + -+ for (i = 0; i < sh->num_entry_point_offsets; i++) -+ controls->entry_point_offsets[i] = sh->entry_point_offset[i]; ++ for (i = 0, offsets = controls->num_entry_point_offsets; i < sh->num_entry_point_offsets; i++) ++ controls->entry_point_offsets[offsets + i] = sh->entry_point_offset[i]; ++ controls->num_entry_point_offsets += sh->num_entry_point_offsets; + + return 0; +} + +static void fill_sps(struct v4l2_ctrl_hevc_sps *ctrl, const HEVCContext *h) +{ -+ const HEVCSPS *sps = h->ps.sps; -+ const HEVCPPS *pps = h->ps.pps; ++ const HEVCPPS *pps = h->pps; ++ const HEVCSPS *sps = pps->sps; + + /* ISO/IEC 23008-2, ITU-T Rec. H.265: Sequence parameter set */ + *ctrl = (struct v4l2_ctrl_hevc_sps) { @@ -2856,50 +3953,56 @@ index 000000000000..57a0d5e78092 + .bit_depth_luma_minus8 = sps->bit_depth - 8, + .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8, + .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4, -+ .sps_max_dec_pic_buffering_minus1 = sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering - 1, -+ .sps_max_num_reorder_pics = sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics, -+ .sps_max_latency_increase_plus1 = sps->temporal_layer[sps->max_sub_layers - 1].max_latency_increase + 1, ++ .sps_max_dec_pic_buffering_minus1 = ++ sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering - 1, ++ .sps_max_num_reorder_pics = ++ sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics, ++ .sps_max_latency_increase_plus1 = ++ sps->temporal_layer[sps->max_sub_layers - 1].max_latency_increase + 1, + .log2_min_luma_coding_block_size_minus3 = sps->log2_min_cb_size - 3, -+ .log2_diff_max_min_luma_coding_block_size = sps->log2_diff_max_min_coding_block_size, ++ .log2_diff_max_min_luma_coding_block_size = ++ sps->log2_diff_max_min_coding_block_size, + .log2_min_luma_transform_block_size_minus2 = sps->log2_min_tb_size - 2, -+ .log2_diff_max_min_luma_transform_block_size = sps->log2_max_trafo_size - sps->log2_min_tb_size, ++ .log2_diff_max_min_luma_transform_block_size = ++ sps->log2_max_trafo_size - sps->log2_min_tb_size, + .max_transform_hierarchy_depth_inter = sps->max_transform_hierarchy_depth_inter, + .max_transform_hierarchy_depth_intra = sps->max_transform_hierarchy_depth_intra, + .pcm_sample_bit_depth_luma_minus1 = sps->pcm.bit_depth - 1, + .pcm_sample_bit_depth_chroma_minus1 = sps->pcm.bit_depth_chroma - 1, + .log2_min_pcm_luma_coding_block_size_minus3 = sps->pcm.log2_min_pcm_cb_size - 3, -+ .log2_diff_max_min_pcm_luma_coding_block_size = sps->pcm.log2_max_pcm_cb_size - sps->pcm.log2_min_pcm_cb_size, ++ .log2_diff_max_min_pcm_luma_coding_block_size = ++ sps->pcm.log2_max_pcm_cb_size - sps->pcm.log2_min_pcm_cb_size, + .num_short_term_ref_pic_sets = sps->nb_st_rps, + .num_long_term_ref_pics_sps = sps->num_long_term_ref_pics_sps, + .chroma_format_idc = sps->chroma_format_idc, + .sps_max_sub_layers_minus1 = sps->max_sub_layers - 1, + }; + -+ if (sps->separate_colour_plane_flag) ++ if (sps->separate_colour_plane) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SEPARATE_COLOUR_PLANE; + -+ if (sps->scaling_list_enable_flag) ++ if (sps->scaling_list_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SCALING_LIST_ENABLED; + -+ if (sps->amp_enabled_flag) ++ if (sps->amp_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_AMP_ENABLED; + + if (sps->sao_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SAMPLE_ADAPTIVE_OFFSET; + -+ if (sps->pcm_enabled_flag) ++ if (sps->pcm_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_PCM_ENABLED; + -+ if (sps->pcm.loop_filter_disable_flag) ++ if (sps->pcm_loop_filter_disabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED; + -+ if (sps->long_term_ref_pics_present_flag) ++ if (sps->long_term_ref_pics_present) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT; + -+ if (sps->sps_temporal_mvp_enabled_flag) ++ if (sps->temporal_mvp_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED; + -+ if (sps->sps_strong_intra_smoothing_enable_flag) ++ if (sps->strong_intra_smoothing_enabled) + ctrl->flags |= V4L2_HEVC_SPS_FLAG_STRONG_INTRA_SMOOTHING_ENABLED; +} + @@ -2908,31 +4011,39 @@ index 000000000000..57a0d5e78092 + av_unused uint32_t size) +{ + const HEVCContext *h = avctx->priv_data; -+ const HEVCPPS *pps = h->ps.pps; -+ const HEVCSPS *sps = h->ps.sps; -+ const ScalingList *sl = pps->scaling_list_data_present_flag ? -+ &pps->scaling_list : -+ sps->scaling_list_enable_flag ? -+ &sps->scaling_list : NULL; -+ V4L2RequestControlsHEVC *controls = h->ref->hwaccel_picture_private; ++ const HEVCPPS *pps = h->pps; ++ const HEVCSPS *sps = pps->sps; ++ V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; ++ V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private; + const SliceHeader *sh = &h->sh; ++ int ret; ++ ++ ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_frame->f); ++ if (ret) ++ return ret; + + fill_sps(&controls->sps, h); -+ fill_dec_params(&controls->dec_params, h); ++ fill_decode_params(&controls->decode_params, h); + -+ if (sl) { -+ for (int i = 0; i < 6; i++) { -+ for (int j = 0; j < 16; j++) -+ controls->scaling_matrix.scaling_list_4x4[i][j] = sl->sl[0][i][j]; -+ for (int j = 0; j < 64; j++) { -+ controls->scaling_matrix.scaling_list_8x8[i][j] = sl->sl[1][i][j]; -+ controls->scaling_matrix.scaling_list_16x16[i][j] = sl->sl[2][i][j]; ++ if (ctx->has_scaling_matrix) { ++ const ScalingList *sl = pps->scaling_list_data_present_flag ? ++ &pps->scaling_list : ++ sps->scaling_list_enabled ? ++ &sps->scaling_list : NULL; ++ if (sl) { ++ for (int i = 0; i < 6; i++) { ++ for (int j = 0; j < 16; j++) ++ controls->scaling_matrix.scaling_list_4x4[i][j] = sl->sl[0][i][j]; ++ for (int j = 0; j < 64; j++) { ++ controls->scaling_matrix.scaling_list_8x8[i][j] = sl->sl[1][i][j]; ++ controls->scaling_matrix.scaling_list_16x16[i][j] = sl->sl[2][i][j]; ++ if (i < 2) ++ controls->scaling_matrix.scaling_list_32x32[i][j] = sl->sl[3][i * 3][j]; ++ } ++ controls->scaling_matrix.scaling_list_dc_coef_16x16[i] = sl->sl_dc[0][i]; + if (i < 2) -+ controls->scaling_matrix.scaling_list_32x32[i][j] = sl->sl[3][i * 3][j]; ++ controls->scaling_matrix.scaling_list_dc_coef_32x32[i] = sl->sl_dc[1][i * 3]; + } -+ controls->scaling_matrix.scaling_list_dc_coef_16x16[i] = sl->sl_dc[0][i]; -+ if (i < 2) -+ controls->scaling_matrix.scaling_list_dc_coef_32x32[i] = sl->sl_dc[1][i * 3]; + } + } + @@ -3025,123 +4136,153 @@ index 000000000000..57a0d5e78092 + controls->pps.row_height_minus1[i] = pps->row_height[i] - 1; + } + -+ controls->first_slice = 1; -+ controls->num_slices = 0; ++ controls->first_slice = true; ++ controls->frame_slice_params = &controls->slice_params; ++ controls->allocated_slice_params = 0; ++ controls->num_slice_params = 0; ++ controls->allocated_entry_point_offsets = 0; + controls->num_entry_point_offsets = 0; + -+ return ff_v4l2_request_reset_frame(avctx, h->ref->frame); ++ return 0; +} + -+static int v4l2_request_hevc_queue_decode(AVCodecContext *avctx, int last_slice) ++static int v4l2_request_hevc_queue_decode(AVCodecContext *avctx, bool last_slice) +{ + const HEVCContext *h = avctx->priv_data; -+ V4L2RequestControlsHEVC *controls = h->ref->hwaccel_picture_private; -+ struct v4l2_ctrl_hevc_slice_params *first_slice_params = &controls->slice_params[0]; + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; -+ int num_controls = 0; ++ V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private; ++ int count = 0; + + struct v4l2_ext_control control[V4L2_HEVC_CONTROLS_MAX] = {}; + -+ control[num_controls++] = (struct v4l2_ext_control) { ++ control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_SPS, + .ptr = &controls->sps, + .size = sizeof(controls->sps), + }; + -+ control[num_controls++] = (struct v4l2_ext_control) { ++ control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_PPS, + .ptr = &controls->pps, + .size = sizeof(controls->pps), + }; + -+ control[num_controls++] = (struct v4l2_ext_control) { ++ control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS, -+ .ptr = &controls->dec_params, -+ .size = sizeof(controls->dec_params), ++ .ptr = &controls->decode_params, ++ .size = sizeof(controls->decode_params), + }; + -+ if (ctx->supports_scaling_matrix) { -+ control[num_controls++] = (struct v4l2_ext_control) { ++ if (ctx->has_scaling_matrix) { ++ control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, + .ptr = &controls->scaling_matrix, + .size = sizeof(controls->scaling_matrix), + }; + } + -+ if (ctx->supports_slices) { -+ control[num_controls++] = (struct v4l2_ext_control) { ++ if (ctx->max_slice_params && controls->num_slice_params) { ++ control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, -+ .ptr = &controls->slice_params, -+ .size = sizeof(*first_slice_params) * controls->num_slices, ++ .ptr = controls->frame_slice_params, ++ .size = sizeof(*controls->frame_slice_params) * ++ FFMIN(controls->num_slice_params, ctx->max_slice_params), + }; + } + -+ //this assumes that decoders supporting entry_point_offsets submit a single slice per request -+ if (ctx->supports_entry_point_offsets && first_slice_params->num_entry_point_offsets > 0) { -+ control[num_controls++] = (struct v4l2_ext_control) { ++ if (ctx->max_entry_point_offsets && controls->num_entry_point_offsets) { ++ control[count++] = (struct v4l2_ext_control) { + .id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS, + .ptr = controls->entry_point_offsets, -+ .size = sizeof(*controls->entry_point_offsets) * first_slice_params->num_entry_point_offsets, ++ .size = sizeof(*controls->entry_point_offsets) * ++ FFMIN(controls->num_entry_point_offsets, ++ ctx->max_entry_point_offsets), + }; + } + + if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED) -+ return ff_v4l2_request_decode_slice(avctx, h->ref->frame, control, num_controls, controls->first_slice, last_slice); ++ return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count, ++ controls->first_slice, last_slice); + -+ return ff_v4l2_request_decode_frame(avctx, h->ref->frame, control, num_controls); ++ return ff_v4l2_request_decode_frame(avctx, &controls->pic, control, count); +} + -+static int v4l2_request_hevc_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) ++static int v4l2_request_hevc_decode_slice(AVCodecContext *avctx, ++ const uint8_t *buffer, uint32_t size) +{ + const HEVCContext *h = avctx->priv_data; -+ V4L2RequestControlsHEVC *controls = h->ref->hwaccel_picture_private; + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; -+ V4L2RequestDescriptor *req = (V4L2RequestDescriptor*)h->ref->frame->data[0]; -+ int ret, slice = FFMIN(controls->num_slices, MAX_SLICES - 1); ++ V4L2RequestControlsHEVC *controls = h->cur_frame->hwaccel_picture_private; ++ const SliceHeader *sh = &h->sh; ++ int ret, slice = controls->num_slice_params; ++ uint32_t extra_size = 0; + -+ if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED && slice) { -+ ret = v4l2_request_hevc_queue_decode(avctx, 0); ++ if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED && ++ (slice >= ctx->max_slice_params || (ctx->max_entry_point_offsets && ++ (controls->num_entry_point_offsets + sh->num_entry_point_offsets > ctx->max_entry_point_offsets)))) { ++ ret = v4l2_request_hevc_queue_decode(avctx, false); + if (ret) + return ret; + -+ ff_v4l2_request_reset_frame(avctx, h->ref->frame); -+ slice = controls->num_slices = 0; -+ controls->first_slice = 0; ++ ff_v4l2_request_reset_picture(avctx, &controls->pic); ++ slice = controls->num_slice_params = 0; ++ controls->num_entry_point_offsets = 0; ++ controls->first_slice = false; + } + -+ ret = v4l2_request_hevc_fill_slice_params(h, controls, slice); -+ if (ret) -+ return ret; -+ + if (ctx->start_code == V4L2_STATELESS_HEVC_START_CODE_ANNEX_B) { -+ ret = ff_v4l2_request_append_output_buffer(avctx, h->ref->frame, nalu_slice_start_code, 3); ++ ret = ff_v4l2_request_append_output(avctx, &controls->pic, ++ nalu_slice_start_code, 3); ++ if (ret) ++ return ret; ++ extra_size = 3; ++ } ++ ++ if (ctx->max_slice_params) { ++ if (slice && controls->allocated_slice_params < slice + 1) { ++ void *slice_params = controls->allocated_slice_params == 0 ? NULL : controls->frame_slice_params; ++ int slices = controls->allocated_slice_params == 0 ? 8 : controls->allocated_slice_params * 2; ++ slice_params = av_realloc_array(slice_params, slices, sizeof(*controls->frame_slice_params)); ++ if (!slice_params) ++ return AVERROR(ENOMEM); ++ if (controls->allocated_slice_params == 0) ++ memcpy(slice_params, controls->frame_slice_params, sizeof(*controls->frame_slice_params)); ++ controls->frame_slice_params = slice_params; ++ controls->allocated_slice_params = slices; ++ } ++ ++ ret = fill_slice_params(controls, slice, !!ctx->max_entry_point_offsets, h); + if (ret) + return ret; + } + -+ ret = ff_v4l2_request_append_output_buffer(avctx, h->ref->frame, buffer, size); ++ ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); + if (ret) + return ret; + -+ controls->slice_params[slice].bit_size = req->output.used * 8; //FIXME -+ controls->num_slices++; ++ if (ctx->max_slice_params) ++ controls->frame_slice_params[slice].bit_size = (size + extra_size) * 8; ++ ++ controls->num_slice_params++; + return 0; +} + +static int v4l2_request_hevc_end_frame(AVCodecContext *avctx) +{ -+ const HEVCContext *h = avctx->priv_data; -+ V4L2RequestControlsHEVC *controls = h->ref->hwaccel_picture_private; -+ int ret; -+ -+ ret = v4l2_request_hevc_queue_decode(avctx, 1); -+ -+ av_freep(&controls->entry_point_offsets); -+ -+ return ret; ++ return v4l2_request_hevc_queue_decode(avctx, true); +} + -+static int v4l2_request_hevc_set_controls(AVCodecContext *avctx) ++static void v4l2_request_hevc_free_frame_priv(FFRefStructOpaque hwctx, void *data) ++{ ++ V4L2RequestControlsHEVC *controls = data; ++ ++ if (controls->allocated_slice_params) ++ av_freep(&controls->frame_slice_params); ++ ++ av_freep(&controls->entry_point_offsets); ++} ++ ++static int v4l2_request_hevc_post_probe(AVCodecContext *avctx) +{ + V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; + int ret; @@ -3150,74 +4291,59 @@ index 000000000000..57a0d5e78092 + { .id = V4L2_CID_STATELESS_HEVC_DECODE_MODE, }, + { .id = V4L2_CID_STATELESS_HEVC_START_CODE, }, + }; ++ struct v4l2_query_ext_ctrl scaling_matrix = { ++ .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, ++ }; + struct v4l2_query_ext_ctrl entry_point_offsets = { + .id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS, + }; + struct v4l2_query_ext_ctrl slice_params = { + .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, + }; -+ struct v4l2_query_ext_ctrl scaling_matrix = { -+ .id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, -+ }; + -+ ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, V4L2_CID_STATELESS_HEVC_DECODE_MODE); ++ ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, ++ V4L2_CID_STATELESS_HEVC_DECODE_MODE); + if (ctx->decode_mode != V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED && + ctx->decode_mode != V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED) { -+ av_log(avctx, AV_LOG_ERROR, "%s: unsupported decode mode, %d\n", __func__, ctx->decode_mode); ++ av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n", ++ ctx->decode_mode); + return AVERROR(EINVAL); + } + -+ ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, V4L2_CID_STATELESS_HEVC_START_CODE); ++ ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, ++ V4L2_CID_STATELESS_HEVC_START_CODE); + if (ctx->start_code != V4L2_STATELESS_HEVC_START_CODE_NONE && + ctx->start_code != V4L2_STATELESS_HEVC_START_CODE_ANNEX_B) { -+ av_log(avctx, AV_LOG_ERROR, "%s: unsupported start code, %d\n", __func__, ctx->start_code); ++ av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n", ++ ctx->start_code); + return AVERROR(EINVAL); + } + -+ ret = ff_v4l2_request_query_control(avctx, &entry_point_offsets); -+ if (ret) -+ ctx->supports_entry_point_offsets = 0; -+ else -+ ctx->supports_entry_point_offsets = 1; -+ -+ ret = ff_v4l2_request_query_control(avctx, &slice_params); -+ if (ret) { -+ ctx->supports_slices = 0; -+ ctx->max_slices = 0; -+ if (ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED) { -+ av_log(avctx, AV_LOG_ERROR, "%s: decoder is slice-based, \ -+ but doesn't support V4L2_CID_STATELESS_HEVC_SLICE_PARAMS control \n", __func__); -+ return AVERROR(EINVAL); -+ } -+ -+ if (ctx->supports_entry_point_offsets) { -+ av_log(avctx, AV_LOG_ERROR, "%s: decoder supports entry_point_offsets, \ -+ but doesn't support V4L2_CID_STATELESS_HEVC_SLICE_PARAMS control \n", __func__); -+ return AVERROR(EINVAL); -+ } -+ } else { -+ ctx->supports_slices = 1; -+ ctx->max_slices = slice_params.dims[0]; -+ if (ctx->max_slices > MAX_SLICES) { -+ av_log(avctx, AV_LOG_ERROR, "%s: unsupported max slices, %u\n", __func__, ctx->max_slices); -+ return AVERROR(EINVAL); -+ } -+ } ++ // TODO: check V4L2_CID_MPEG_VIDEO_HEVC_PROFILE control ++ // TODO: check V4L2_CID_MPEG_VIDEO_HEVC_LEVEL control + + ret = ff_v4l2_request_query_control(avctx, &scaling_matrix); -+ if (ret) -+ ctx->supports_scaling_matrix = 0; ++ if (!ret) ++ ctx->has_scaling_matrix = true; + else -+ ctx->supports_scaling_matrix = 1; ++ ctx->has_scaling_matrix = false; + -+ av_log(avctx, AV_LOG_DEBUG, "%s: decoder is %s and supports slices %d, supports entry_point_offsets: %d supports scaling_matrix: %d max slices: %u\n", -+ __func__, -+ ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED ? "slice based" : "frame based", -+ ctx->supports_slices, -+ ctx->supports_entry_point_offsets, -+ ctx->supports_scaling_matrix, -+ ctx->max_slices -+ ); ++ ret = ff_v4l2_request_query_control(avctx, &entry_point_offsets); ++ if (!ret) ++ ctx->max_entry_point_offsets = FFMAX(entry_point_offsets.dims[0], 1); ++ else ++ ctx->max_entry_point_offsets = 0; ++ ++ ret = ff_v4l2_request_query_control(avctx, &slice_params); ++ if (!ret) ++ ctx->max_slice_params = FFMAX(slice_params.dims[0], 1); ++ else ++ ctx->max_slice_params = 0; ++ ++ av_log(ctx, AV_LOG_VERBOSE, "%s-based decoder with SLICE_PARAMS=%u, " ++ "ENTRY_POINT_OFFSETS=%u and SCALING_MATRIX=%d controls\n", ++ ctx->decode_mode == V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED ? "slice" : "frame", ++ ctx->max_slice_params, ctx->max_entry_point_offsets, ctx->has_scaling_matrix); + + control[0].value = ctx->decode_mode; + control[1].value = ctx->start_code; @@ -3227,9 +4353,9 @@ index 000000000000..57a0d5e78092 + +static int v4l2_request_hevc_init(AVCodecContext *avctx) +{ ++ V4L2RequestContextHEVC *ctx = avctx->internal->hwaccel_priv_data; + const HEVCContext *h = avctx->priv_data; + struct v4l2_ctrl_hevc_sps sps; -+ int ret; + + struct v4l2_ext_control control[] = { + { @@ -3241,98 +4367,101 @@ index 000000000000..57a0d5e78092 + + fill_sps(&sps, h); + -+ ret = ff_v4l2_request_init(avctx, V4L2_PIX_FMT_HEVC_SLICE, 4 * 1024 * 1024, control, FF_ARRAY_ELEMS(control)); -+ if (ret) -+ return ret; -+ -+ return v4l2_request_hevc_set_controls(avctx); ++ ctx->base.post_probe = v4l2_request_hevc_post_probe; ++ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_HEVC_SLICE, ++ 4 * 1024 * 1024, ++ control, FF_ARRAY_ELEMS(control)); +} + -+const AVHWAccel ff_hevc_v4l2request_hwaccel = { -+ .name = "hevc_v4l2request", -+ .type = AVMEDIA_TYPE_VIDEO, -+ .id = AV_CODEC_ID_HEVC, -+ .pix_fmt = AV_PIX_FMT_DRM_PRIME, -+ .start_frame = v4l2_request_hevc_start_frame, -+ .decode_slice = v4l2_request_hevc_decode_slice, -+ .end_frame = v4l2_request_hevc_end_frame, ++const FFHWAccel ff_hevc_v4l2request_hwaccel = { ++ .p.name = "hevc_v4l2request", ++ .p.type = AVMEDIA_TYPE_VIDEO, ++ .p.id = AV_CODEC_ID_HEVC, ++ .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, ++ .start_frame = v4l2_request_hevc_start_frame, ++ .decode_slice = v4l2_request_hevc_decode_slice, ++ .end_frame = v4l2_request_hevc_end_frame, ++ .flush = ff_v4l2_request_flush, ++ .free_frame_priv = v4l2_request_hevc_free_frame_priv, + .frame_priv_data_size = sizeof(V4L2RequestControlsHEVC), -+ .init = v4l2_request_hevc_init, -+ .uninit = ff_v4l2_request_uninit, -+ .priv_data_size = sizeof(V4L2RequestContextHEVC), -+ .frame_params = ff_v4l2_request_frame_params, -+ .caps_internal = HWACCEL_CAP_ASYNC_SAFE, ++ .init = v4l2_request_hevc_init, ++ .uninit = ff_v4l2_request_uninit, ++ .priv_data_size = sizeof(V4L2RequestContextHEVC), ++ .frame_params = ff_v4l2_request_frame_params, +}; -From 344839ebfdd9163a978f57e4b61f78a0501fcf83 Mon Sep 17 00:00:00 2001 +From 63716a8632a02eb3c9d7715a5dd0bf99d4296d48 Mon Sep 17 00:00:00 2001 From: Boris Brezillon -Date: Thu, 12 Dec 2019 16:13:55 +0100 -Subject: [PATCH 08/11] Add V4L2 request API VP9 hwaccel +Date: Wed, 22 May 2019 14:46:58 +0200 +Subject: [PATCH 09/11] avcodec: Add V4L2 Request API vp8 hwaccel +Add a V4L2 Request API hwaccel for VP8. + +Co-developed-by: Ezequiel Garcia +Signed-off-by: Ezequiel Garcia Signed-off-by: Boris Brezillon -Signed-off-by: Jernej Skrabec +Co-developed-by: Jonas Karlman +Signed-off-by: Jonas Karlman --- configure | 3 + libavcodec/Makefile | 1 + libavcodec/hwaccels.h | 1 + - libavcodec/v4l2_request_vp9.c | 282 ++++++++++++++++++++++++++++++++++ - libavcodec/vp9.c | 190 ++++++++++++++++------- - libavcodec/vp9dec.h | 4 + - libavcodec/vp9shared.h | 1 + - 7 files changed, 428 insertions(+), 54 deletions(-) - create mode 100644 libavcodec/v4l2_request_vp9.c + libavcodec/v4l2_request_vp8.c | 235 ++++++++++++++++++++++++++++++++++ + libavcodec/vp8.c | 6 + + 5 files changed, 246 insertions(+) + create mode 100644 libavcodec/v4l2_request_vp8.c diff --git a/configure b/configure -index e6a4f0013218..f85e16d272d5 100755 +index 7a2901d960..3b1288eb98 100755 --- a/configure +++ b/configure -@@ -3119,6 +3119,8 @@ vp9_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_VP9" - vp9_dxva2_hwaccel_select="vp9_decoder" - vp9_nvdec_hwaccel_deps="nvdec" - vp9_nvdec_hwaccel_select="vp9_decoder" -+vp9_v4l2request_hwaccel_deps="v4l2_request" -+vp9_v4l2request_hwaccel_select="vp9_decoder" - vp9_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferVP9_bit_depth" - vp9_vaapi_hwaccel_select="vp9_decoder" - vp9_vdpau_hwaccel_deps="vdpau VdpPictureInfoVP9" -@@ -6899,6 +6901,7 @@ check_cc h264_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_H264_SLICE;" - check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_HEVC_SLICE;" - check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_MPEG2_SLICE;" - check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_VP8_FRAME;" -+check_cc vp9_v4l2_request linux/videodev2.h "int i = V4L2_PIX_FMT_VP9_FRAME;" - - check_headers sys/videoio.h - test_code cc sys/videoio.h "struct v4l2_frmsizeenum vfse; vfse.discrete.width = 0;" && enable_sanitized struct_v4l2_frmivalenum_discrete +@@ -3271,6 +3271,8 @@ vc1_vdpau_hwaccel_deps="vdpau" + vc1_vdpau_hwaccel_select="vc1_decoder" + vp8_nvdec_hwaccel_deps="nvdec" + vp8_nvdec_hwaccel_select="vp8_decoder" ++vp8_v4l2request_hwaccel_deps="v4l2_request vp8_v4l2_request" ++vp8_v4l2request_hwaccel_select="vp8_decoder" + vp8_vaapi_hwaccel_deps="vaapi" + vp8_vaapi_hwaccel_select="vp8_decoder" + vp9_d3d11va_hwaccel_deps="d3d11va DXVA_PicParams_VP9" +@@ -7213,6 +7215,7 @@ if enabled v4l2_request; then + check_cc hevc_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_HEVC_SPS" + check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" + check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" ++ check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP8_FRAME" + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns + check_pkg_config libudev libudev libudev.h udev_new + check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx diff --git a/libavcodec/Makefile b/libavcodec/Makefile -index cd08740e75f1..e2c957763a9e 100644 +index fa78523b76..3fa9822594 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile -@@ -1030,6 +1030,7 @@ OBJS-$(CONFIG_VP8_VAAPI_HWACCEL) += vaapi_vp8.o +@@ -1057,6 +1057,7 @@ OBJS-$(CONFIG_VC1_QSV_HWACCEL) += qsvdec.o + OBJS-$(CONFIG_VC1_VAAPI_HWACCEL) += vaapi_vc1.o + OBJS-$(CONFIG_VC1_VDPAU_HWACCEL) += vdpau_vc1.o + OBJS-$(CONFIG_VP8_NVDEC_HWACCEL) += nvdec_vp8.o ++OBJS-$(CONFIG_VP8_V4L2REQUEST_HWACCEL) += v4l2_request_vp8.o + OBJS-$(CONFIG_VP8_VAAPI_HWACCEL) += vaapi_vp8.o OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL) += dxva2_vp9.o OBJS-$(CONFIG_VP9_DXVA2_HWACCEL) += dxva2_vp9.o - OBJS-$(CONFIG_VP9_NVDEC_HWACCEL) += nvdec_vp9.o -+OBJS-$(CONFIG_VP9_V4L2REQUEST_HWACCEL) += v4l2_request_vp9.o - OBJS-$(CONFIG_VP9_VAAPI_HWACCEL) += vaapi_vp9.o - OBJS-$(CONFIG_VP9_VDPAU_HWACCEL) += vdpau_vp9.o - OBJS-$(CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) += videotoolbox_vp9.o diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h -index e4e4abc0606a..53f4f61fc517 100644 +index ad145fc07c..cb2cdb3893 100644 --- a/libavcodec/hwaccels.h +++ b/libavcodec/hwaccels.h -@@ -76,6 +76,7 @@ extern const AVHWAccel ff_vp9_d3d11va_hwaccel; - extern const AVHWAccel ff_vp9_d3d11va2_hwaccel; - extern const AVHWAccel ff_vp9_dxva2_hwaccel; - extern const AVHWAccel ff_vp9_nvdec_hwaccel; -+extern const AVHWAccel ff_vp9_v4l2request_hwaccel; - extern const AVHWAccel ff_vp9_vaapi_hwaccel; - extern const AVHWAccel ff_vp9_vdpau_hwaccel; - extern const AVHWAccel ff_vp9_videotoolbox_hwaccel; -diff --git a/libavcodec/v4l2_request_vp9.c b/libavcodec/v4l2_request_vp9.c +@@ -76,6 +76,7 @@ extern const struct FFHWAccel ff_vc1_nvdec_hwaccel; + extern const struct FFHWAccel ff_vc1_vaapi_hwaccel; + extern const struct FFHWAccel ff_vc1_vdpau_hwaccel; + extern const struct FFHWAccel ff_vp8_nvdec_hwaccel; ++extern const struct FFHWAccel ff_vp8_v4l2request_hwaccel; + extern const struct FFHWAccel ff_vp8_vaapi_hwaccel; + extern const struct FFHWAccel ff_vp9_d3d11va_hwaccel; + extern const struct FFHWAccel ff_vp9_d3d11va2_hwaccel; +diff --git a/libavcodec/v4l2_request_vp8.c b/libavcodec/v4l2_request_vp8.c new file mode 100644 -index 000000000000..ec0300f66dbe +index 0000000000..551b6dc75f --- /dev/null -+++ b/libavcodec/v4l2_request_vp9.c -@@ -0,0 +1,282 @@ ++++ b/libavcodec/v4l2_request_vp8.c +@@ -0,0 +1,235 @@ +/* + * This file is part of FFmpeg. + * @@ -3351,12 +4480,350 @@ index 000000000000..ec0300f66dbe + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + ++#include "config.h" ++ ++#include "hwaccel_internal.h" ++#include "hwconfig.h" ++#include "v4l2_request.h" ++#include "vp8.h" ++ ++typedef struct V4L2RequestControlsVP8 { ++ V4L2RequestPictureContext pic; ++ struct v4l2_ctrl_vp8_frame frame; ++} V4L2RequestControlsVP8; ++ ++static int v4l2_request_vp8_start_frame(AVCodecContext *avctx, ++ const uint8_t *buffer, ++ av_unused uint32_t size) ++{ ++ const VP8Context *s = avctx->priv_data; ++ V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; ++ struct v4l2_ctrl_vp8_frame *ctrl = &controls->frame; ++ unsigned int header_size = 3 + 7 * s->keyframe; ++ const uint8_t *data = buffer + header_size; ++ int ret, i, j, k; ++ ++ ret = ff_v4l2_request_start_frame(avctx, &controls->pic, ++ s->framep[VP8_FRAME_CURRENT]->tf.f); ++ if (ret) ++ return ret; ++ ++ *ctrl = (struct v4l2_ctrl_vp8_frame) { ++ .lf = { ++ .sharpness_level = s->filter.sharpness, ++ .level = s->filter.level, ++ }, ++ ++ .quant = { ++ .y_ac_qi = s->quant.yac_qi, ++ .y_dc_delta = s->quant.ydc_delta, ++ .y2_dc_delta = s->quant.y2dc_delta, ++ .y2_ac_delta = s->quant.y2ac_delta, ++ .uv_dc_delta = s->quant.uvdc_delta, ++ .uv_ac_delta = s->quant.uvac_delta, ++ }, ++ ++ .coder_state = { ++ .range = s->coder_state_at_header_end.range, ++ .value = s->coder_state_at_header_end.value, ++ .bit_count = s->coder_state_at_header_end.bit_count, ++ }, ++ ++ .width = avctx->width, ++ .height = avctx->height, ++ ++ .horizontal_scale = 0, /* scale not supported by FFmpeg */ ++ .vertical_scale = 0, /* scale not supported by FFmpeg */ ++ ++ .version = s->profile & 0x3, ++ .prob_skip_false = s->prob->mbskip, ++ .prob_intra = s->prob->intra, ++ .prob_last = s->prob->last, ++ .prob_gf = s->prob->golden, ++ .num_dct_parts = s->num_coeff_partitions, ++ ++ .first_part_size = s->header_partition_size, ++ .first_part_header_bits = (8 * (s->coder_state_at_header_end.input - data) - ++ s->coder_state_at_header_end.bit_count - 8), ++ }; ++ ++ for (i = 0; i < 4; i++) { ++ ctrl->segment.quant_update[i] = s->segmentation.base_quant[i]; ++ ctrl->segment.lf_update[i] = s->segmentation.filter_level[i]; ++ } ++ ++ for (i = 0; i < 3; i++) ++ ctrl->segment.segment_probs[i] = s->prob->segmentid[i]; ++ ++ if (s->segmentation.enabled) ++ ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_ENABLED; ++ ++ if (s->segmentation.update_map) ++ ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_UPDATE_MAP; ++ ++ if (s->segmentation.update_feature_data) ++ ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_UPDATE_FEATURE_DATA; ++ ++ if (!s->segmentation.absolute_vals) ++ ctrl->segment.flags |= V4L2_VP8_SEGMENT_FLAG_DELTA_VALUE_MODE; ++ ++ for (i = 0; i < 4; i++) { ++ ctrl->lf.ref_frm_delta[i] = s->lf_delta.ref[i]; ++ ctrl->lf.mb_mode_delta[i] = s->lf_delta.mode[i + MODE_I4x4]; ++ } ++ ++ if (s->lf_delta.enabled) ++ ctrl->lf.flags |= V4L2_VP8_LF_ADJ_ENABLE; ++ ++ if (s->lf_delta.update) ++ ctrl->lf.flags |= V4L2_VP8_LF_DELTA_UPDATE; ++ ++ if (s->filter.simple) ++ ctrl->lf.flags |= V4L2_VP8_LF_FILTER_TYPE_SIMPLE; ++ ++ if (s->keyframe) { ++ static const uint8_t keyframe_y_mode_probs[4] = { ++ 145, 156, 163, 128 ++ }; ++ static const uint8_t keyframe_uv_mode_probs[3] = { ++ 142, 114, 183 ++ }; ++ ++ memcpy(ctrl->entropy.y_mode_probs, keyframe_y_mode_probs, 4); ++ memcpy(ctrl->entropy.uv_mode_probs, keyframe_uv_mode_probs, 3); ++ } else { ++ for (i = 0; i < 4; i++) ++ ctrl->entropy.y_mode_probs[i] = s->prob->pred16x16[i]; ++ for (i = 0; i < 3; i++) ++ ctrl->entropy.uv_mode_probs[i] = s->prob->pred8x8c[i]; ++ } ++ for (i = 0; i < 2; i++) ++ for (j = 0; j < 19; j++) ++ ctrl->entropy.mv_probs[i][j] = s->prob->mvc[i][j]; ++ ++ for (i = 0; i < 4; i++) { ++ for (j = 0; j < 8; j++) { ++ static const int coeff_bands_inverse[8] = { ++ 0, 1, 2, 3, 5, 6, 4, 15 ++ }; ++ int coeff_pos = coeff_bands_inverse[j]; ++ ++ for (k = 0; k < 3; k++) { ++ memcpy(ctrl->entropy.coeff_probs[i][j][k], ++ s->prob->token[i][coeff_pos][k], 11); ++ } ++ } ++ } ++ ++ for (i = 0; i < 8; i++) ++ ctrl->dct_part_sizes[i] = s->coeff_partition_size[i]; ++ ++ if (s->framep[VP8_FRAME_PREVIOUS]) ++ ctrl->last_frame_ts = ++ ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_PREVIOUS]->tf.f); ++ if (s->framep[VP8_FRAME_GOLDEN]) ++ ctrl->golden_frame_ts = ++ ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_GOLDEN]->tf.f); ++ if (s->framep[VP8_FRAME_ALTREF]) ++ ctrl->alt_frame_ts = ++ ff_v4l2_request_get_capture_timestamp(s->framep[VP8_FRAME_ALTREF]->tf.f); ++ ++ if (s->keyframe) ++ ctrl->flags |= V4L2_VP8_FRAME_FLAG_KEY_FRAME; ++ ++ if (s->profile & 0x4) ++ ctrl->flags |= V4L2_VP8_FRAME_FLAG_EXPERIMENTAL; ++ ++ if (!s->invisible) ++ ctrl->flags |= V4L2_VP8_FRAME_FLAG_SHOW_FRAME; ++ ++ if (s->mbskip_enabled) ++ ctrl->flags |= V4L2_VP8_FRAME_FLAG_MB_NO_SKIP_COEFF; ++ ++ if (s->sign_bias[VP8_FRAME_GOLDEN]) ++ ctrl->flags |= V4L2_VP8_FRAME_FLAG_SIGN_BIAS_GOLDEN; ++ ++ if (s->sign_bias[VP8_FRAME_ALTREF]) ++ ctrl->flags |= V4L2_VP8_FRAME_FLAG_SIGN_BIAS_ALT; ++ ++ return 0; ++} ++ ++static int v4l2_request_vp8_decode_slice(AVCodecContext *avctx, ++ const uint8_t *buffer, uint32_t size) ++{ ++ const VP8Context *s = avctx->priv_data; ++ V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; ++ ++ return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); ++} ++ ++static int v4l2_request_vp8_end_frame(AVCodecContext *avctx) ++{ ++ const VP8Context *s = avctx->priv_data; ++ V4L2RequestControlsVP8 *controls = s->framep[VP8_FRAME_CURRENT]->hwaccel_picture_private; ++ ++ struct v4l2_ext_control control[] = { ++ { ++ .id = V4L2_CID_STATELESS_VP8_FRAME, ++ .ptr = &controls->frame, ++ .size = sizeof(controls->frame), ++ }, ++ }; ++ ++ return ff_v4l2_request_decode_frame(avctx, &controls->pic, ++ control, FF_ARRAY_ELEMS(control)); ++} ++ ++static int v4l2_request_vp8_init(AVCodecContext *avctx) ++{ ++ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_VP8_FRAME, ++ 2 * 1024 * 1024, ++ NULL, 0); ++} ++ ++const FFHWAccel ff_vp8_v4l2request_hwaccel = { ++ .p.name = "vp8_v4l2request", ++ .p.type = AVMEDIA_TYPE_VIDEO, ++ .p.id = AV_CODEC_ID_VP8, ++ .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, ++ .start_frame = v4l2_request_vp8_start_frame, ++ .decode_slice = v4l2_request_vp8_decode_slice, ++ .end_frame = v4l2_request_vp8_end_frame, ++ .flush = ff_v4l2_request_flush, ++ .frame_priv_data_size = sizeof(V4L2RequestControlsVP8), ++ .init = v4l2_request_vp8_init, ++ .uninit = ff_v4l2_request_uninit, ++ .priv_data_size = sizeof(V4L2RequestContext), ++ .frame_params = ff_v4l2_request_frame_params, ++}; +diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c +index d6df018655..04b16cc2e3 100644 +--- a/libavcodec/vp8.c ++++ b/libavcodec/vp8.c +@@ -184,6 +184,9 @@ static enum AVPixelFormat get_pixel_format(VP8Context *s) + #endif + #if CONFIG_VP8_NVDEC_HWACCEL + AV_PIX_FMT_CUDA, ++#endif ++#if CONFIG_VP8_V4L2REQUEST_HWACCEL ++ AV_PIX_FMT_DRM_PRIME, + #endif + AV_PIX_FMT_YUV420P, + AV_PIX_FMT_NONE, +@@ -2980,6 +2983,9 @@ const FFCodec ff_vp8_decoder = { + #endif + #if CONFIG_VP8_NVDEC_HWACCEL + HWACCEL_NVDEC(vp8), ++#endif ++#if CONFIG_VP8_V4L2REQUEST_HWACCEL ++ HWACCEL_V4L2REQUEST(vp8), + #endif + NULL + }, + +From 96c22879bbbfcbd57cd73a165ab6b1fb49663481 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 12 Dec 2019 16:13:55 +0100 +Subject: [PATCH 10/11] avcodec: Add V4L2 Request API vp9 hwaccel + +FIXME: split patch + +Signed-off-by: Boris Brezillon +Co-developed-by: Jernej Skrabec +Signed-off-by: Jernej Skrabec +Co-developed-by: Jonas Karlman +Signed-off-by: Jonas Karlman +--- + configure | 3 + + libavcodec/Makefile | 1 + + libavcodec/hwaccels.h | 1 + + libavcodec/v4l2_request_vp9.c | 288 ++++++++++++++++++++++++++++++++++ + libavcodec/vp9.c | 187 +++++++++++++++------- + libavcodec/vp9dec.h | 4 + + libavcodec/vp9shared.h | 1 + + 7 files changed, 431 insertions(+), 54 deletions(-) + create mode 100644 libavcodec/v4l2_request_vp9.c + +diff --git a/configure b/configure +index 3b1288eb98..2904b67ef7 100755 +--- a/configure ++++ b/configure +@@ -3285,6 +3285,8 @@ vp9_dxva2_hwaccel_deps="dxva2 DXVA_PicParams_VP9" + vp9_dxva2_hwaccel_select="vp9_decoder" + vp9_nvdec_hwaccel_deps="nvdec" + vp9_nvdec_hwaccel_select="vp9_decoder" ++vp9_v4l2request_hwaccel_deps="v4l2_request vp9_v4l2_request" ++vp9_v4l2request_hwaccel_select="vp9_decoder" + vp9_vaapi_hwaccel_deps="vaapi VADecPictureParameterBufferVP9_bit_depth" + vp9_vaapi_hwaccel_select="vp9_decoder" + vp9_vdpau_hwaccel_deps="vdpau VdpPictureInfoVP9" +@@ -7216,6 +7218,7 @@ if enabled v4l2_request; then + check_cc mpeg2_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_MPEG2_SEQUENCE" + check_cc v4l2_m2m_hold_capture_buf linux/videodev2.h "int i = V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF" + check_cc vp8_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP8_FRAME" ++ check_cc vp9_v4l2_request linux/videodev2.h "int i = V4L2_CID_STATELESS_VP9_FRAME" + check_func_headers "linux/media.h linux/videodev2.h" v4l2_timeval_to_ns + check_pkg_config libudev libudev libudev.h udev_new + check_struct linux/videodev2.h "struct v4l2_ctrl_hevc_decode_params" num_delta_pocs_of_ref_rps_idx +diff --git a/libavcodec/Makefile b/libavcodec/Makefile +index 3fa9822594..95c77d7bec 100644 +--- a/libavcodec/Makefile ++++ b/libavcodec/Makefile +@@ -1063,6 +1063,7 @@ OBJS-$(CONFIG_VP9_D3D11VA_HWACCEL) += dxva2_vp9.o + OBJS-$(CONFIG_VP9_DXVA2_HWACCEL) += dxva2_vp9.o + OBJS-$(CONFIG_VP9_D3D12VA_HWACCEL) += dxva2_vp9.o d3d12va_vp9.o + OBJS-$(CONFIG_VP9_NVDEC_HWACCEL) += nvdec_vp9.o ++OBJS-$(CONFIG_VP9_V4L2REQUEST_HWACCEL) += v4l2_request_vp9.o + OBJS-$(CONFIG_VP9_VAAPI_HWACCEL) += vaapi_vp9.o + OBJS-$(CONFIG_VP9_VDPAU_HWACCEL) += vdpau_vp9.o + OBJS-$(CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) += videotoolbox_vp9.o +diff --git a/libavcodec/hwaccels.h b/libavcodec/hwaccels.h +index cb2cdb3893..8571cc7e8b 100644 +--- a/libavcodec/hwaccels.h ++++ b/libavcodec/hwaccels.h +@@ -83,6 +83,7 @@ extern const struct FFHWAccel ff_vp9_d3d11va2_hwaccel; + extern const struct FFHWAccel ff_vp9_d3d12va_hwaccel; + extern const struct FFHWAccel ff_vp9_dxva2_hwaccel; + extern const struct FFHWAccel ff_vp9_nvdec_hwaccel; ++extern const struct FFHWAccel ff_vp9_v4l2request_hwaccel; + extern const struct FFHWAccel ff_vp9_vaapi_hwaccel; + extern const struct FFHWAccel ff_vp9_vdpau_hwaccel; + extern const struct FFHWAccel ff_vp9_videotoolbox_hwaccel; +diff --git a/libavcodec/v4l2_request_vp9.c b/libavcodec/v4l2_request_vp9.c +new file mode 100644 +index 0000000000..41ea4e37f5 +--- /dev/null ++++ b/libavcodec/v4l2_request_vp9.c +@@ -0,0 +1,288 @@ ++/* ++ * This file is part of FFmpeg. ++ * ++ * FFmpeg is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2.1 of the License, or (at your option) any later version. ++ * ++ * FFmpeg is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ * Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with FFmpeg; if not, write to the Free Software ++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ */ ++ ++#include "config.h" ++ ++#include "hwaccel_internal.h" +#include "hwconfig.h" +#include "v4l2_request.h" +#include "vp9dec.h" + +typedef struct V4L2RequestControlsVP9 { -+ struct v4l2_ctrl_vp9_frame decode_params; ++ V4L2RequestPictureContext pic; ++ struct v4l2_ctrl_vp9_frame frame; + struct v4l2_ctrl_vp9_compressed_hdr chp; +} V4L2RequestControlsVP9; + @@ -3412,7 +4879,7 @@ index 000000000000..ec0300f66dbe +static void fill_frame(struct v4l2_ctrl_vp9_frame *dec_params, AVCodecContext *avctx) +{ + const VP9Context *s = avctx->priv_data; -+ const ThreadFrame *ref; ++ AVFrame *ref; + + memset(dec_params, 0, sizeof(*dec_params)); + @@ -3453,15 +4920,15 @@ index 000000000000..ec0300f66dbe + //dec_params->render_width_minus_1 = avctx->width - 1; + //dec_params->render_height_minus_1 = avctx->height - 1; + -+ ref = &s->s.refs[s->s.h.refidx[0]]; -+ if (ref->f && ref->f->buf[0]) -+ dec_params->last_frame_ts = ff_v4l2_request_get_capture_timestamp(ref->f); -+ ref = &s->s.refs[s->s.h.refidx[1]]; -+ if (ref->f && ref->f->buf[0]) -+ dec_params->golden_frame_ts = ff_v4l2_request_get_capture_timestamp(ref->f); -+ ref = &s->s.refs[s->s.h.refidx[2]]; -+ if (ref->f && ref->f->buf[0]) -+ dec_params->alt_frame_ts = ff_v4l2_request_get_capture_timestamp(ref->f); ++ ref = s->s.refs[s->s.h.refidx[0]].f; ++ if (ref && ref->buf[0]) ++ dec_params->last_frame_ts = ff_v4l2_request_get_capture_timestamp(ref); ++ ref = s->s.refs[s->s.h.refidx[1]].f; ++ if (ref && ref->buf[0]) ++ dec_params->golden_frame_ts = ff_v4l2_request_get_capture_timestamp(ref); ++ ref = s->s.refs[s->s.h.refidx[2]].f; ++ if (ref && ref->buf[0]) ++ dec_params->alt_frame_ts = ff_v4l2_request_get_capture_timestamp(ref); + + if (s->s.h.signbias[0]) + dec_params->ref_frame_sign_bias |= V4L2_VP9_SIGN_BIAS_LAST; @@ -3533,37 +5000,41 @@ index 000000000000..ec0300f66dbe + av_unused const uint8_t *buffer, + av_unused uint32_t size) +{ -+ const VP9Context *s = avctx->priv_data; -+ const VP9Frame *f = &s->s.frames[CUR_FRAME]; ++ const VP9SharedContext *h = avctx->priv_data; ++ const VP9Frame *f = &h->frames[CUR_FRAME]; + V4L2RequestControlsVP9 *controls = f->hwaccel_picture_private; ++ int ret; ++ ++ ret = ff_v4l2_request_start_frame(avctx, &controls->pic, f->tf.f); ++ if (ret) ++ return ret; + + v4l2_request_vp9_set_frame_ctx(avctx); + -+ fill_frame(&controls->decode_params, avctx); ++ fill_frame(&controls->frame, avctx); + -+ return ff_v4l2_request_reset_frame(avctx, f->tf.f); ++ return 0; +} + -+static int v4l2_request_vp9_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) ++static int v4l2_request_vp9_decode_slice(AVCodecContext *avctx, ++ const uint8_t *buffer, uint32_t size) +{ -+ const VP9Context *s = avctx->priv_data; -+ const VP9Frame *f = &s->s.frames[CUR_FRAME]; ++ const VP9SharedContext *h = avctx->priv_data; ++ V4L2RequestControlsVP9 *controls = h->frames[CUR_FRAME].hwaccel_picture_private; + -+ return ff_v4l2_request_append_output_buffer(avctx, f->tf.f, buffer, size); ++ return ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); +} + +static int v4l2_request_vp9_end_frame(AVCodecContext *avctx) +{ -+ const VP9Context *s = avctx->priv_data; -+ const VP9Frame *f = &s->s.frames[CUR_FRAME]; -+ V4L2RequestControlsVP9 *controls = f->hwaccel_picture_private; -+ int ret; ++ const VP9SharedContext *h = avctx->priv_data; ++ V4L2RequestControlsVP9 *controls = h->frames[CUR_FRAME].hwaccel_picture_private; + + struct v4l2_ext_control control[] = { + { + .id = V4L2_CID_STATELESS_VP9_FRAME, -+ .ptr = &controls->decode_params, -+ .size = sizeof(controls->decode_params), ++ .ptr = &controls->frame, ++ .size = sizeof(controls->frame), + }, + { + .id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR, @@ -3572,14 +5043,8 @@ index 000000000000..ec0300f66dbe + }, + }; + -+ ret = ff_v4l2_request_decode_frame(avctx, f->tf.f, control, FF_ARRAY_ELEMS(control)); -+ if (ret) -+ return ret; -+ -+ if (!s->s.h.refreshctx) -+ return 0; -+ -+ return 0; ++ return ff_v4l2_request_decode_frame(avctx, &controls->pic, ++ control, FF_ARRAY_ELEMS(control)); +} + +static int v4l2_request_vp9_init(AVCodecContext *avctx) @@ -3597,57 +5062,51 @@ index 000000000000..ec0300f66dbe + fill_frame(&frame, avctx); + + // TODO: check V4L2_CID_MPEG_VIDEO_VP9_PROFILE -+ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_VP9_FRAME, 3 * 1024 * 1024, control, FF_ARRAY_ELEMS(control)); ++ // TODO: check V4L2_CID_MPEG_VIDEO_VP9_LEVEL ++ ++ return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_VP9_FRAME, ++ 3 * 1024 * 1024, ++ control, FF_ARRAY_ELEMS(control)); +} + -+const AVHWAccel ff_vp9_v4l2request_hwaccel = { -+ .name = "vp9_v4l2request", -+ .type = AVMEDIA_TYPE_VIDEO, -+ .id = AV_CODEC_ID_VP9, -+ .pix_fmt = AV_PIX_FMT_DRM_PRIME, -+ .start_frame = v4l2_request_vp9_start_frame, -+ .decode_slice = v4l2_request_vp9_decode_slice, -+ .end_frame = v4l2_request_vp9_end_frame, ++const FFHWAccel ff_vp9_v4l2request_hwaccel = { ++ .p.name = "vp9_v4l2request", ++ .p.type = AVMEDIA_TYPE_VIDEO, ++ .p.id = AV_CODEC_ID_VP9, ++ .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, ++ .start_frame = v4l2_request_vp9_start_frame, ++ .decode_slice = v4l2_request_vp9_decode_slice, ++ .end_frame = v4l2_request_vp9_end_frame, ++ .flush = ff_v4l2_request_flush, + .frame_priv_data_size = sizeof(V4L2RequestControlsVP9), -+ .init = v4l2_request_vp9_init, -+ .uninit = ff_v4l2_request_uninit, -+ .priv_data_size = sizeof(V4L2RequestContext), -+ .frame_params = ff_v4l2_request_frame_params, -+ .caps_internal = HWACCEL_CAP_ASYNC_SAFE, ++ .init = v4l2_request_vp9_init, ++ .uninit = ff_v4l2_request_uninit, ++ .priv_data_size = sizeof(V4L2RequestContext), ++ .frame_params = ff_v4l2_request_frame_params, +}; diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c -index 7c0a24644670..3927c5d54076 100644 +index 8ede2e2eb3..17b611bfdd 100644 --- a/libavcodec/vp9.c +++ b/libavcodec/vp9.c -@@ -185,6 +185,7 @@ static int update_size(AVCodecContext *avctx, int w, int h) - #define HWACCEL_MAX (CONFIG_VP9_DXVA2_HWACCEL + \ +@@ -163,6 +163,7 @@ static int update_size(AVCodecContext *avctx, int w, int h) CONFIG_VP9_D3D11VA_HWACCEL * 2 + \ + CONFIG_VP9_D3D12VA_HWACCEL + \ CONFIG_VP9_NVDEC_HWACCEL + \ + CONFIG_VP9_V4L2REQUEST_HWACCEL + \ CONFIG_VP9_VAAPI_HWACCEL + \ CONFIG_VP9_VDPAU_HWACCEL + \ CONFIG_VP9_VIDEOTOOLBOX_HWACCEL) -@@ -213,6 +214,9 @@ static int update_size(AVCodecContext *avctx, int w, int h) - #if CONFIG_VP9_NVDEC_HWACCEL - *fmtp++ = AV_PIX_FMT_CUDA; +@@ -202,6 +203,9 @@ static int update_size(AVCodecContext *avctx, int w, int h) #endif + #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL + *fmtp++ = AV_PIX_FMT_VIDEOTOOLBOX; ++#endif +#if CONFIG_VP9_V4L2REQUEST_HWACCEL + *fmtp++ = AV_PIX_FMT_DRM_PRIME; -+#endif - #if CONFIG_VP9_VAAPI_HWACCEL - *fmtp++ = AV_PIX_FMT_VAAPI; #endif -@@ -227,6 +231,9 @@ static int update_size(AVCodecContext *avctx, int w, int h) - #if CONFIG_VP9_NVDEC_HWACCEL - *fmtp++ = AV_PIX_FMT_CUDA; - #endif -+#if CONFIG_VP9_V4L2REQUEST_HWACCEL -+ *fmtp++ = AV_PIX_FMT_DRM_PRIME; -+#endif - #if CONFIG_VP9_VAAPI_HWACCEL - *fmtp++ = AV_PIX_FMT_VAAPI; - #endif -@@ -387,7 +394,7 @@ static av_always_inline int inv_recenter_nonneg(int v, int m) + break; + case AV_PIX_FMT_YUV420P12: +@@ -375,7 +379,7 @@ static av_always_inline int inv_recenter_nonneg(int v, int m) } // differential forward probability updates @@ -3656,7 +5115,7 @@ index 7c0a24644670..3927c5d54076 100644 { static const uint8_t inv_map_table[255] = { 7, 20, 33, 46, 59, 72, 85, 98, 111, 124, 137, 150, 163, 176, -@@ -441,8 +448,13 @@ static int update_prob(VPXRangeCoder *c, int p) +@@ -429,8 +433,13 @@ static int update_prob(VPXRangeCoder *c, int p) av_assert2(d < FF_ARRAY_ELEMS(inv_map_table)); } @@ -3672,7 +5131,7 @@ index 7c0a24644670..3927c5d54076 100644 } static int read_colorspace_details(AVCodecContext *avctx) -@@ -708,7 +720,8 @@ static int decode_frame_header(AVCodecContext *avctx, +@@ -697,7 +706,8 @@ static int decode_frame_header(AVCodecContext *avctx, get_bits(&s->gb, 8) : 255; } @@ -3682,7 +5141,7 @@ index 7c0a24644670..3927c5d54076 100644 s->s.h.segmentation.absolute_vals = get_bits1(&s->gb); for (i = 0; i < 8; i++) { if ((s->s.h.segmentation.feat[i].q_enabled = get_bits1(&s->gb))) -@@ -908,6 +921,8 @@ static int decode_frame_header(AVCodecContext *avctx, +@@ -903,6 +913,8 @@ static int decode_frame_header(AVCodecContext *avctx, * as explicit copies if the fw update is missing (and skip the copy upon * fw update)? */ s->prob.p = s->prob_ctx[c].p; @@ -3691,7 +5150,7 @@ index 7c0a24644670..3927c5d54076 100644 // txfm updates if (s->s.h.lossless) { -@@ -919,18 +934,25 @@ static int decode_frame_header(AVCodecContext *avctx, +@@ -914,18 +926,25 @@ static int decode_frame_header(AVCodecContext *avctx, if (s->s.h.txfmmode == TX_SWITCHABLE) { for (i = 0; i < 2; i++) @@ -3725,7 +5184,7 @@ index 7c0a24644670..3927c5d54076 100644 } } -@@ -942,15 +964,18 @@ static int decode_frame_header(AVCodecContext *avctx, +@@ -937,15 +956,18 @@ static int decode_frame_header(AVCodecContext *avctx, for (k = 0; k < 2; k++) for (l = 0; l < 6; l++) for (m = 0; m < 6; m++) { @@ -3747,7 +5206,7 @@ index 7c0a24644670..3927c5d54076 100644 } memcpy(&p[3], ff_vp9_model_pareto8[p[2]], 8); } -@@ -973,25 +998,37 @@ static int decode_frame_header(AVCodecContext *avctx, +@@ -968,25 +990,37 @@ static int decode_frame_header(AVCodecContext *avctx, // mode updates for (i = 0; i < 3; i++) @@ -3793,7 +5252,7 @@ index 7c0a24644670..3927c5d54076 100644 if (s->s.h.allowcompinter) { s->s.h.comppredmode = vp89_rac_get(&s->c); -@@ -999,92 +1036,134 @@ static int decode_frame_header(AVCodecContext *avctx, +@@ -994,92 +1028,134 @@ static int decode_frame_header(AVCodecContext *avctx, s->s.h.comppredmode += vp89_rac_get(&s->c); if (s->s.h.comppredmode == PRED_SWITCHABLE) for (i = 0; i < 5; i++) @@ -3959,21 +5418,21 @@ index 7c0a24644670..3927c5d54076 100644 } } } -@@ -1906,6 +1985,9 @@ const FFCodec ff_vp9_decoder = { - #if CONFIG_VP9_VDPAU_HWACCEL - HWACCEL_VDPAU(vp9), +@@ -1870,6 +1946,9 @@ const FFCodec ff_vp9_decoder = { #endif -+#if CONFIG_VP9_V4L2REQUEST_HWACCEL -+ HWACCEL_V4L2REQUEST(vp9), -+#endif #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL HWACCEL_VIDEOTOOLBOX(vp9), ++#endif ++#if CONFIG_VP9_V4L2REQUEST_HWACCEL ++ HWACCEL_V4L2REQUEST(vp9), #endif + NULL + }, diff --git a/libavcodec/vp9dec.h b/libavcodec/vp9dec.h -index de7aba045851..5935ba622738 100644 +index 81dc801052..99df1cb730 100644 --- a/libavcodec/vp9dec.h +++ b/libavcodec/vp9dec.h -@@ -135,6 +135,10 @@ typedef struct VP9Context { +@@ -134,6 +134,10 @@ typedef struct VP9Context { ProbContext p; uint8_t coef[4][2][2][6][6][11]; } prob; @@ -3985,10 +5444,10 @@ index de7aba045851..5935ba622738 100644 // contextual (above) cache uint8_t *above_partition_ctx; diff --git a/libavcodec/vp9shared.h b/libavcodec/vp9shared.h -index 543a496df8fc..a5028d4b399d 100644 +index 8a450c26a6..37e389e9e1 100644 --- a/libavcodec/vp9shared.h +++ b/libavcodec/vp9shared.h -@@ -137,6 +137,7 @@ typedef struct VP9BitstreamHeader { +@@ -136,6 +136,7 @@ typedef struct VP9BitstreamHeader { uint8_t temporal; uint8_t absolute_vals; uint8_t update_map; @@ -3997,158 +5456,31 @@ index 543a496df8fc..a5028d4b399d 100644 uint8_t pred_prob[3]; struct { -From ad5f9b06ae829d0046ae8f3f9035702a9cd11d5f Mon Sep 17 00:00:00 2001 -From: Jonas Karlman -Date: Mon, 29 Apr 2019 22:08:59 +0000 -Subject: [PATCH 09/11] HACK: hwcontext_drm: do not require drm device +From 904a85173fab816bb3c30652300efa93f2333657 Mon Sep 17 00:00:00 2001 +From: Alex Bee +Date: Sun, 15 Dec 2024 01:16:13 +0100 +Subject: [PATCH 11/11] HACK: avcodec: Don't check VIDIOC_ENUM_FRAMESIZES + step_width / step_height -Signed-off-by: Jonas Karlman +Currently required, since hantro and rkvdec drivers expose wrong values. + +Signed-off-by: Alex Bee --- - libavutil/hwcontext_drm.c | 5 +++++ - 1 file changed, 5 insertions(+) + libavcodec/v4l2_request_probe.c | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) -diff --git a/libavutil/hwcontext_drm.c b/libavutil/hwcontext_drm.c -index 7a9fdbd263d4..6297d1f9b613 100644 ---- a/libavutil/hwcontext_drm.c -+++ b/libavutil/hwcontext_drm.c -@@ -53,6 +53,11 @@ static int drm_device_create(AVHWDeviceContext *hwdev, const char *device, - AVDRMDeviceContext *hwctx = hwdev->hwctx; - drmVersionPtr version; +diff --git a/libavcodec/v4l2_request_probe.c b/libavcodec/v4l2_request_probe.c +index 941042ec04..b3b1af3744 100644 +--- a/libavcodec/v4l2_request_probe.c ++++ b/libavcodec/v4l2_request_probe.c +@@ -258,9 +258,7 @@ static int v4l2_request_try_framesize(AVCodecContext *avctx, + } else if ((frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE || + frmsize.type == V4L2_FRMSIZE_TYPE_CONTINUOUS) && + avctx->coded_width <= frmsize.stepwise.max_width && +- avctx->coded_height <= frmsize.stepwise.max_height && +- avctx->coded_width % frmsize.stepwise.step_width == 0 && +- avctx->coded_height % frmsize.stepwise.step_height == 0) { ++ avctx->coded_height <= frmsize.stepwise.max_height) { + return 0; + } -+ if (device == NULL) { -+ hwctx->fd = -1; -+ return 0; -+ } -+ - hwctx->fd = open(device, O_RDWR); - if (hwctx->fd < 0) - return AVERROR(errno); - -From 6eb2656d8c739c962a2fb3057da869e1d2bc8b1d Mon Sep 17 00:00:00 2001 -From: Jonas Karlman -Date: Fri, 15 May 2020 16:54:05 +0000 -Subject: [PATCH 10/11] WIP: add NV15 and NV20 support - -Signed-off-by: Jonas Karlman ---- - libavcodec/h264_slice.c | 9 ++++++--- - libavcodec/v4l2_request.c | 7 +++++++ - 2 files changed, 13 insertions(+), 3 deletions(-) - -diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c -index 3d0d45b2a3f5..e8005181b681 100644 ---- a/libavcodec/h264_slice.c -+++ b/libavcodec/h264_slice.c -@@ -802,6 +802,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) - #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL - if (h->avctx->colorspace != AVCOL_SPC_RGB) - *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX; -+#endif -+#if CONFIG_H264_V4L2REQUEST_HWACCEL -+ *fmt++ = AV_PIX_FMT_DRM_PRIME; - #endif - if (CHROMA444(h)) { - if (h->avctx->colorspace == AVCOL_SPC_RGB) { -@@ -845,6 +848,9 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) - #if CONFIG_H264_VIDEOTOOLBOX_HWACCEL - if (h->avctx->colorspace != AVCOL_SPC_RGB) - *fmt++ = AV_PIX_FMT_VIDEOTOOLBOX; -+#endif -+#if CONFIG_H264_V4L2REQUEST_HWACCEL -+ *fmt++ = AV_PIX_FMT_DRM_PRIME; - #endif - if (CHROMA444(h)) { - if (h->avctx->colorspace == AVCOL_SPC_RGB) -@@ -868,9 +874,6 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback) - #endif - #if CONFIG_H264_VAAPI_HWACCEL - *fmt++ = AV_PIX_FMT_VAAPI; --#endif --#if CONFIG_H264_V4L2REQUEST_HWACCEL -- *fmt++ = AV_PIX_FMT_DRM_PRIME; - #endif - if (h->avctx->codec->pix_fmts) - choices = h->avctx->codec->pix_fmts; -diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c -index 595a53e6d4d8..8d2a2481b821 100644 ---- a/libavcodec/v4l2_request.c -+++ b/libavcodec/v4l2_request.c -@@ -196,6 +196,13 @@ static const struct { - } v4l2_request_capture_pixelformats[] = { - { V4L2_PIX_FMT_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_LINEAR }, - { V4L2_PIX_FMT_SUNXI_TILED_NV12, AV_PIX_FMT_NV12, DRM_FORMAT_NV12, DRM_FORMAT_MOD_ALLWINNER_TILED }, -+#if defined(V4L2_PIX_FMT_NV15) && defined(DRM_FORMAT_NV15) -+ { V4L2_PIX_FMT_NV15, AV_PIX_FMT_NONE, DRM_FORMAT_NV15, DRM_FORMAT_MOD_LINEAR }, -+#endif -+ { V4L2_PIX_FMT_NV16, AV_PIX_FMT_NV16, DRM_FORMAT_NV16, DRM_FORMAT_MOD_LINEAR }, -+#if defined(V4L2_PIX_FMT_NV20) && defined(DRM_FORMAT_NV20) -+ { V4L2_PIX_FMT_NV20, AV_PIX_FMT_NONE, DRM_FORMAT_NV20, DRM_FORMAT_MOD_LINEAR }, -+#endif - }; - - static int v4l2_request_set_drm_descriptor(V4L2RequestDescriptor *req, struct v4l2_format *format) - -From 34c3a37d8fbc808286c1f55b14230491d1107105 Mon Sep 17 00:00:00 2001 -From: Jernej Skrabec -Date: Mon, 9 Oct 2023 22:19:18 +0200 -Subject: [PATCH 11/11] v4l2 request api: add new codecs - ---- - libavcodec/v4l2_request.c | 32 ++++++++++++++++++++++++++++---- - 1 file changed, 28 insertions(+), 4 deletions(-) - -diff --git a/libavcodec/v4l2_request.c b/libavcodec/v4l2_request.c -index 8d2a2481b821..813c1b381eaf 100644 ---- a/libavcodec/v4l2_request.c -+++ b/libavcodec/v4l2_request.c -@@ -202,6 +202,27 @@ static const struct { - { V4L2_PIX_FMT_NV16, AV_PIX_FMT_NV16, DRM_FORMAT_NV16, DRM_FORMAT_MOD_LINEAR }, - #if defined(V4L2_PIX_FMT_NV20) && defined(DRM_FORMAT_NV20) - { V4L2_PIX_FMT_NV20, AV_PIX_FMT_NONE, DRM_FORMAT_NV20, DRM_FORMAT_MOD_LINEAR }, -+#endif -+ { V4L2_PIX_FMT_P010, AV_PIX_FMT_P010, DRM_FORMAT_P010, DRM_FORMAT_MOD_LINEAR }, -+#if defined(V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT) -+ { -+ .pixelformat = V4L2_PIX_FMT_YUV420_10_AFBC_16X16_SPLIT, -+ .sw_format = AV_PIX_FMT_NONE, -+ .drm_format = DRM_FORMAT_YUV420_10BIT, -+ .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | -+ AFBC_FORMAT_MOD_SPARSE | -+ AFBC_FORMAT_MOD_SPLIT), -+ }, -+#endif -+#if defined(V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT) -+ { -+ .pixelformat = V4L2_PIX_FMT_YUV420_8_AFBC_16X16_SPLIT, -+ .sw_format = AV_PIX_FMT_NONE, -+ .drm_format = DRM_FORMAT_YUV420_8BIT, -+ .format_modifier = DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | -+ AFBC_FORMAT_MOD_SPARSE | -+ AFBC_FORMAT_MOD_SPLIT), -+ }, - #endif - }; - -@@ -228,15 +249,18 @@ static int v4l2_request_set_drm_descriptor(V4L2RequestDescriptor *req, struct v4 - desc->objects[0].size = req->capture.size; - - desc->nb_layers = 1; -- layer->nb_planes = 2; -+ layer->nb_planes = 1; - - layer->planes[0].object_index = 0; - layer->planes[0].offset = 0; - layer->planes[0].pitch = V4L2_TYPE_IS_MULTIPLANAR(format->type) ? format->fmt.pix_mp.plane_fmt[0].bytesperline : format->fmt.pix.bytesperline; - -- layer->planes[1].object_index = 0; -- layer->planes[1].offset = layer->planes[0].pitch * (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? format->fmt.pix_mp.height : format->fmt.pix.height); -- layer->planes[1].pitch = layer->planes[0].pitch; -+ if (!fourcc_mod_is_vendor(desc->objects[0].format_modifier, ARM)) { -+ layer->nb_planes = 2; -+ layer->planes[1].object_index = 0; -+ layer->planes[1].offset = layer->planes[0].pitch * (V4L2_TYPE_IS_MULTIPLANAR(format->type) ? format->fmt.pix_mp.height : format->fmt.pix.height); -+ layer->planes[1].pitch = layer->planes[0].pitch; -+ } - - return 0; - }