/* * Copyright (C) 2026 Philip Langdale * * Based on vf_framerate - Copyright (C) 2012 Mark Himsley * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ /** * @file * Frame rate up-conversion filter that synthesises intermediate frames using * the NVIDIA Vulkan optical flow extension (VK_NV_optical_flow). */ #include "libavutil/avassert.h" #include "libavutil/eval.h" #include "libavutil/internal.h" #include "libavutil/mem.h" #include "libavutil/opt.h" #include "libavutil/pixdesc.h" #include "vulkan_filter.h" #include "filters.h" #include "video.h" extern const unsigned char ff_fruc_grayscale_comp_spv_data[]; extern const unsigned int ff_fruc_grayscale_comp_spv_len; extern const unsigned char ff_fruc_interpolate_comp_spv_data[]; extern const unsigned int ff_fruc_interpolate_comp_spv_len; static const char *const var_names[] = { "source_fps", NULL }; enum var_name { VAR_SOURCE_FPS, VARS_NB }; typedef struct GrayscalePushData { float luma_weights[4][4]; ///< per-plane RGB->Y weights (dotted with each plane's texel) int32_t planes; ///< number of input planes sampled per frame } GrayscalePushData; typedef struct InterpolatePushData { float t; int32_t planes; float luma_weights[4][4]; ///< per-plane RGB->Y weights, matching the grayscale pass float plane_size[4][2]; ///< visible texel extent of each plane } InterpolatePushData; /* Double-buffer the per-pair optical flow resources so one pair's flow execution * overlaps the previous pair's interpolations instead of stalling on shared * images. The session bakes in its image bindings, so each slot owns its session, * grayscale and flow images. Two suffice: the flow engine is serial. */ #define FRUC_NB_SLOTS 2 typedef struct FRUCFlowSlot { VkOpticalFlowSessionNV session; VkImage gray_img[2]; ///< grayscale inputs (INPUT, REFERENCE) VkDeviceMemory gray_mem[2]; VkImageView gray_view[2]; VkImage flow_img[2]; ///< [0] forward, [1] backward VkDeviceMemory flow_mem[2]; VkImageView flow_view[2]; ///< native (SFIXED5) view, bound to the OF session VkImageView flow_sint_view[2]; ///< R16G16_SINT reinterpret view for sampling /* sem_interp value reached by the pair that last used this slot; the next * pair to reuse it waits here before overwriting the flow images. Zero (the * initial value) is satisfied immediately, covering the first use. */ uint64_t interp_done; } FRUCFlowSlot; typedef struct FRUCVulkanContext { FFVulkanContext vkctx; AVVulkanDeviceQueueFamily *qf; ///< compute queue family AVVulkanDeviceQueueFamily *qf_of; ///< optical flow queue family FFVkExecPool e; ///< compute execution pool FFVkExecPool e_of; ///< optical flow execution pool FFVulkanShader grayscale; FFVulkanShader interpolate; VkSampler sampler; ///< linear sampler for the video planes VkSampler flow_sampler; ///< nearest sampler for the flow vectors /* Timeline semaphores order the pipelined cross-queue submissions and make * each stage's writes visible to the next. Timeline is required for two * reasons: * * Each optical flow source pair may yield multiple interpolations, and * each interpolation should wait on the flow. Multi-wait requires timeline * semaphores * * We don't know how many interpolations will be done from a single optical * flow pair ahead of time, so we cannot simply signal completion after the * last one. Instead we increment a timeline semaphore after each one and * then the next flow calculation waits on the final timeline value. */ VkSemaphore sem_gray; ///< grayscale (compute) -> optical flow VkSemaphore sem_flow; ///< optical flow -> interpolation (compute) VkSemaphore sem_interp; ///< interpolation reads -> next pair optical flow uint64_t gen; ///< source pair generation (sem_gray/sem_flow value) uint64_t interp_value; ///< monotonic interpolation counter (sem_interp value) /* Optical flow session parameters (images live per-slot, see slots[]). */ VkOpticalFlowGridSizeFlagsNV grid_bit; int grid_size; VkFormat input_format; ///< grayscale input format VkFormat flow_format; ///< flow vector format /* Tuning options. */ int perf_level; ///< VkOpticalFlowPerformanceLevelNV int opt_grid_size; ///< requested grid in pixels (0 = finest) int width; ///< luma width int height; ///< luma height int flow_width; int flow_height; float luma_weights[4][4]; ///< RGB->Y weights for the grayscale pass int gray_planes; ///< number of input planes the grayscale pass samples /* Double-buffered optical flow resources, indexed by (gen % FRUC_NB_SLOTS). */ FRUCFlowSlot slots[FRUC_NB_SLOTS]; int flow_valid; ///< flow computed for current (f0, f1) pair // parameters char *requested_frame_rate; ///< output fps as an expression AVRational dest_frame_rate; ///< output frames per second AVRational srce_time_base; ///< timebase of source AVRational dest_time_base; ///< timebase of destination AVFrame *work; AVFrame *f0; ///< last frame AVFrame *f1; ///< current frame int64_t pts0; ///< last frame pts in dest_time_base int64_t pts1; ///< current frame pts in dest_time_base int64_t delta; ///< pts1 to pts0 delta int flush; ///< 1 if the filter is being flushed int64_t start_pts; ///< pts of the first output frame int64_t n; ///< output frame counter } FRUCVulkanContext; #define OFFSET(x) offsetof(FRUCVulkanContext, x) #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) static const AVOption fruc_vulkan_options[] = { { "fps", "A string describing the desired output frame rate", OFFSET(requested_frame_rate), AV_OPT_TYPE_STRING, { .str = "60" }, 0, 0, FLAGS }, { "perf", "Optical flow performance level (quality versus speed)", OFFSET(perf_level), AV_OPT_TYPE_INT, { .i64 = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV }, VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV, VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_FAST_NV, FLAGS, .unit = "perf" }, { "slow", "Highest quality, slowest", 0, AV_OPT_TYPE_CONST, { .i64 = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV }, 0, 0, FLAGS, .unit = "perf" }, { "medium", "Balanced quality and speed", 0, AV_OPT_TYPE_CONST, { .i64 = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_MEDIUM_NV }, 0, 0, FLAGS, .unit = "perf" }, { "fast", "Lowest quality, fastest", 0, AV_OPT_TYPE_CONST, { .i64 = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_FAST_NV }, 0, 0, FLAGS, .unit = "perf" }, { "grid", "Optical flow output grid size in pixels (coarser is faster)", OFFSET(opt_grid_size), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 8, FLAGS, .unit = "grid" }, { "auto", "Finest grid the device supports", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, .unit = "grid" }, { "1", "1x1", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, .unit = "grid" }, { "2", "2x2", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, .unit = "grid" }, { "4", "4x4", 0, AV_OPT_TYPE_CONST, { .i64 = 4 }, 0, 0, FLAGS, .unit = "grid" }, { "8", "8x8", 0, AV_OPT_TYPE_CONST, { .i64 = 8 }, 0, 0, FLAGS, .unit = "grid" }, { NULL } }; AVFILTER_DEFINE_CLASS(fruc_vulkan); static VkFormat pick_of_format(FRUCVulkanContext *s, VkOpticalFlowUsageFlagsNV usage, VkFormat preferred) { FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; VkOpticalFlowImageFormatInfoNV info = { .sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV, .usage = usage, }; VkOpticalFlowImageFormatPropertiesNV *props; VkFormat result = VK_FORMAT_UNDEFINED; uint32_t count = 0; vk->GetPhysicalDeviceOpticalFlowImageFormatsNV(vkctx->hwctx->phys_dev, &info, &count, NULL); if (!count) return VK_FORMAT_UNDEFINED; props = av_calloc(count, sizeof(*props)); if (!props) return VK_FORMAT_UNDEFINED; for (uint32_t i = 0; i < count; i++) props[i].sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_PROPERTIES_NV; vk->GetPhysicalDeviceOpticalFlowImageFormatsNV(vkctx->hwctx->phys_dev, &info, &count, props); result = props[0].format; for (uint32_t i = 0; i < count; i++) { av_log(s, AV_LOG_VERBOSE, "Optical flow usage 0x%x supports format %d\n", usage, props[i].format); if (props[i].format == preferred) { result = preferred; break; } } av_free(props); return result; } static int create_of_image(FRUCVulkanContext *s, VkImage *img, VkDeviceMemory *mem, VkImageView *view, VkFormat format, int width, int height, VkOpticalFlowUsageFlagsNV of_usage, VkImageUsageFlags usage, VkImageCreateFlags create_flags) { FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; AVVulkanDeviceContext *hwctx = vkctx->hwctx; VkResult ret; int err; VkOpticalFlowImageFormatInfoNV of_info = { .sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV, .usage = of_usage, }; VkImageViewCreateInfo view_info = { .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .viewType = VK_IMAGE_VIEW_TYPE_2D, .format = format, .components = ff_comp_identity_map, .subresourceRange = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .levelCount = 1, .layerCount = 1, }, }; /* Exclusive sharing: the images are used by both the compute and the * optical flow queue family, and maintenance9 - which init_filter() * requires - preserves their contents across the two without explicit * queue family ownership transfers. */ err = ff_vk_image_create(vkctx, img, mem, width, height, format, 1, VK_IMAGE_TILING_OPTIMAL, usage, create_flags, &of_info); if (err < 0) return err; view_info.image = *img; ret = vk->CreateImageView(hwctx->act_dev, &view_info, hwctx->alloc, view); if (ret != VK_SUCCESS) { av_log(s, AV_LOG_ERROR, "Failed to create optical flow image view: %s\n", ff_vk_ret2str(ret)); return AVERROR_EXTERNAL; } return 0; } /* Transition the persistent optical flow images to VK_IMAGE_LAYOUT_GENERAL, * which is the layout they remain in for the lifetime of the filter. */ static int init_image_layouts(FRUCVulkanContext *s) { FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; FFVkExecContext *exec = ff_vk_exec_get(vkctx, &s->e); VkImageMemoryBarrier2 bar[4 * FRUC_NB_SLOTS]; int nb_bar = 0; int err; err = ff_vk_exec_start(vkctx, exec); if (err < 0) return err; for (int slot = 0; slot < FRUC_NB_SLOTS; slot++) { FRUCFlowSlot *fs = &s->slots[slot]; VkImage imgs[4] = { fs->gray_img[0], fs->gray_img[1], fs->flow_img[0], fs->flow_img[1] }; for (int i = 0; i < 4; i++) { bar[nb_bar++] = (VkImageMemoryBarrier2) { .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .srcAccessMask = 0, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = imgs[i], .subresourceRange = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .levelCount = 1, .layerCount = 1, }, }; } } vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .pImageMemoryBarriers = bar, .imageMemoryBarrierCount = nb_bar, }); err = ff_vk_exec_submit(vkctx, exec); if (err < 0) return err; ff_vk_exec_wait(vkctx, exec); return 0; } static int packed_rgb_channel(const AVPixFmtDescriptor *desc, VkFormat vkfmt, int comp) { const AVComponentDescriptor *c = &desc->comp[comp]; switch (vkfmt) { /* These are all sampled as their logical components, whatever order the * pix fmt uses to lay them out in host memory, so the component index is * already the channel. */ case VK_FORMAT_A2R10G10B10_UNORM_PACK32: case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_B8G8R8_UNORM: case VK_FORMAT_B8G8R8A8_UNORM: return comp; default: return c->offset / ((c->depth + 7) / 8); } } static int packed_luma_channel(const AVPixFmtDescriptor *desc, VkFormat vkfmt) { const AVComponentDescriptor *c = &desc->comp[0]; switch (vkfmt) { /* xv30 packs V, Y, U, so luma is the logical G channel. */ case VK_FORMAT_A2R10G10B10_UNORM_PACK32: return 1; default: return c->offset / ((c->depth + 7) / 8); } } static av_cold int check_sw_format(AVFilterContext *avctx, enum AVPixelFormat sw_format) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sw_format); if (!desc || !av_vkfmt_from_pixfmt(sw_format)) return AVERROR(EINVAL); if (desc->flags & AV_PIX_FMT_FLAG_BAYER) { av_log(avctx, AV_LOG_ERROR, "Bayer input (%s) is not supported\n", desc->name); return AVERROR(ENOTSUP); } if (!(desc->flags & AV_PIX_FMT_FLAG_RGB) && desc->nb_components > 1 && !(desc->flags & AV_PIX_FMT_FLAG_PLANAR) && (desc->log2_chroma_w || desc->log2_chroma_h)) { av_log(avctx, AV_LOG_ERROR, "Subsampled packed YUV input (%s) is not " "supported\n", desc->name); return AVERROR(ENOTSUP); } /* Everything is sampled and stored through FF_VK_REP_FLOAT views. Formats * with more than 16 bits of integer per component (rgb96, rgba128, gray32, * gbrap32) have no float representation at all, so the image views would * fail to be created. */ if (desc->comp[0].depth > 16 && !(desc->flags & AV_PIX_FMT_FLAG_FLOAT)) { av_log(avctx, AV_LOG_ERROR, "Input format %s has no floating point " "shader representation\n", desc->name); return AVERROR(ENOTSUP); } return 0; } /* Whether an optimally tiled image last used by queue family "from" keeps its * contents when used by queue family "to" without an explicit ownership * transfer. */ static int qf_transfer_preserves(FFVulkanContext *vkctx, uint32_t from, uint32_t to) { uint32_t mask; if (from >= (uint32_t)vkctx->tot_nb_qfs || to >= 32) return 0; mask = vkctx->ownership_props[from].optimalImageTransferToQueueFamilies; return !!(mask & (1U << to)); } static av_cold int init_filter(AVFilterContext *avctx) { int err; FRUCVulkanContext *s = avctx->priv; FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; const int planes = av_pix_fmt_count_planes(vkctx->output_format); VkOpticalFlowGridSizeFlagsNV grids; VkResult ret; /* The core Vulkan support covers more formats than we can actually support * in the filter, so reject anything we can't handle up front. */ RET(check_sw_format(avctx, vkctx->output_format)); s->width = vkctx->output_width; s->height = vkctx->output_height; /* Optical flow tracks luma. YUV carries it directly, RGB has it derived so the * flow follows brightness. The value only feeds the flow engine and is never * written out, so a fixed BT.709 matrix suffices for all RGB inputs. */ { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(vkctx->output_format); const VkFormat *vkfmts = av_vkfmt_from_pixfmt(vkctx->output_format); static const float bt709[3] = { 0.2126f, 0.7152f, 0.0722f }; /* R, G, B */ memset(s->luma_weights, 0, sizeof(s->luma_weights)); s->gray_planes = 1; if (desc->flags & AV_PIX_FMT_FLAG_PLANAR) { if (desc->flags & AV_PIX_FMT_FLAG_RGB) { /* One component per plane: weight each plane by its component's * coefficient (comp[c] is R, G, B for c = 0, 1, 2). */ s->gray_planes = av_pix_fmt_count_planes(vkctx->output_format); for (int c = 0; c < 3; c++) s->luma_weights[desc->comp[c].plane][0] = bt709[c]; } else { /* Planar and semi-planar YUV: plane 0 is luma already. */ s->luma_weights[0][0] = 1.0f; } } else if (desc->nb_components > 1) { /* Packed: every component shares plane 0's texel, so the weights * select channels rather than planes. */ if (desc->flags & AV_PIX_FMT_FLAG_RGB) { for (int c = 0; c < 3; c++) s->luma_weights[0][packed_rgb_channel(desc, vkfmts[0], c)] = bt709[c]; } else { /* Packed 4:4:4 YUV: luma is a single component, but not * necessarily the first channel. */ s->luma_weights[0][packed_luma_channel(desc, vkfmts[0])] = 1.0f; } } else { /* Gray: the one channel is luma. */ s->luma_weights[0][0] = 1.0f; } /* Special handling is required for formats that store fewer than * 16bits of data in 16bits of storage. Some of these formats align the * bits to the LSB end, and if these values are interpreted directly as * 16bit values, they will be incorrect. While we could imagine passing * the depth/shift information separately to the shader, we can apply * the adjustment to the luma weights instead. * * Adjustment is not done if shift+depth == 16. In this case, the data * is MSB aligned, and should be treated as a 16bit value. */ for (int p = 0; p < s->gray_planes; p++) { const AVComponentDescriptor *comp = NULL; float scale; if (vkfmts[p] != VK_FORMAT_R16_UNORM) continue; for (int c = 0; c < desc->nb_components; c++) { if (desc->comp[c].plane == p) { comp = &desc->comp[c]; break; } } if (!comp) return AVERROR_BUG; if (comp->shift + comp->depth == 16) continue; scale = 65535.0f / (((1U << comp->depth) - 1U) << comp->shift); for (int c = 0; c < 4; c++) s->luma_weights[p][c] *= scale; } } if (!(vkctx->extensions & FF_VK_EXT_OPTICAL_FLOW)) { av_log(avctx, AV_LOG_ERROR, "Vulkan device does not support the " "VK_NV_optical_flow extension\n"); return AVERROR(ENOTSUP); } /* The extension being enabled is not enough; its feature has to have been * enabled at device creation too. Our own device setup does that, but a * device handed to us by the caller may not have. */ { const VkPhysicalDeviceOpticalFlowFeaturesNV *of; of = ff_vk_find_struct(vkctx->hwctx->device_features.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPTICAL_FLOW_FEATURES_NV); if (!of || !of->opticalFlow) { av_log(avctx, AV_LOG_ERROR, "Vulkan device was created without the " "opticalFlow feature enabled\n"); return AVERROR(ENOTSUP); } } /* The optical flow images must be passed between the compute and optical * flow queue families; maintenance9 allows us to avoid the explicit * ownership transfers that are otherwise required. */ if (!(vkctx->extensions & FF_VK_EXT_MAINTENANCE_9)) { av_log(avctx, AV_LOG_ERROR, "Vulkan device does not support the " "VK_KHR_maintenance9 extension\n"); return AVERROR(ENOTSUP); } { const VkPhysicalDeviceMaintenance9FeaturesKHR *m9; m9 = ff_vk_find_struct(vkctx->hwctx->device_features.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_9_FEATURES_KHR); if (!m9 || !m9->maintenance9) { av_log(avctx, AV_LOG_ERROR, "Vulkan device was created without the " "maintenance9 feature enabled\n"); return AVERROR(ENOTSUP); } } s->qf = ff_vk_qf_find(vkctx, VK_QUEUE_COMPUTE_BIT, 0); if (!s->qf) { av_log(avctx, AV_LOG_ERROR, "Device has no compute queues\n"); return AVERROR(ENOTSUP); } s->qf_of = ff_vk_qf_find(vkctx, VK_QUEUE_OPTICAL_FLOW_BIT_NV, 0); if (!s->qf_of) { av_log(avctx, AV_LOG_ERROR, "Device has no optical flow queues\n"); return AVERROR(ENOTSUP); } /* Even with maintenance9 enabled, we can't assume that we can do implicit * ownership transfers between the queues we care about; the driver doesn't * have to support this, so we must check for declared support. */ if (s->qf_of->idx != s->qf->idx && (!qf_transfer_preserves(vkctx, s->qf->idx, s->qf_of->idx) || !qf_transfer_preserves(vkctx, s->qf_of->idx, s->qf->idx))) { av_log(avctx, AV_LOG_ERROR, "The compute (%d) and optical flow (%d) queue " "families cannot exchange optimally tiled images without explicit " "queue family ownership transfers\n", s->qf->idx, s->qf_of->idx); return AVERROR(ENOTSUP); } RET(ff_vk_exec_pool_init(vkctx, s->qf, &s->e, FF_VK_DEFAULT_EXEC_CONTEXTS, 0, 0, 0, NULL)); /* One optical flow context per slot so that the optical flow execution for * the next frame pair can be recorded and submitted without first * host-waiting the previous pair's execution to retire its command buffer. */ RET(ff_vk_exec_pool_init(vkctx, s->qf_of, &s->e_of, FRUC_NB_SLOTS, 0, 0, 0, NULL)); RET(ff_vk_init_sampler(vkctx, &s->sampler, 0, VK_FILTER_LINEAR)); /* Flow is sampled through an integer view and its format has no linear * filtering support, so use nearest. Requires normalised coords and as we * have only one mip level, the mipmap filtering mode is irrelevant. */ RET(ff_vk_init_sampler(vkctx, &s->flow_sampler, 0, VK_FILTER_NEAREST)); { VkSemaphoreTypeCreateInfo sem_type_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = 0, }; VkSemaphoreCreateInfo sem_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &sem_type_info, }; if (vk->CreateSemaphore(vkctx->hwctx->act_dev, &sem_info, vkctx->hwctx->alloc, &s->sem_gray) != VK_SUCCESS || vk->CreateSemaphore(vkctx->hwctx->act_dev, &sem_info, vkctx->hwctx->alloc, &s->sem_flow) != VK_SUCCESS || vk->CreateSemaphore(vkctx->hwctx->act_dev, &sem_info, vkctx->hwctx->alloc, &s->sem_interp) != VK_SUCCESS) { av_log(avctx, AV_LOG_ERROR, "Failed to create synchronization semaphores\n"); return AVERROR_EXTERNAL; } } /* Select the optical flow image formats and grid size. */ s->input_format = pick_of_format(s, VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV, VK_FORMAT_R8_UNORM); if (s->input_format != VK_FORMAT_R8_UNORM) { av_log(avctx, AV_LOG_ERROR, "Optical flow R8 input format unavailable\n"); return AVERROR(ENOTSUP); } /* The engine emits flow vectors as signed fixed point (SFIXED5); theoretically * it could be something else, but this is all we've seen on real hardware. */ s->flow_format = pick_of_format(s, VK_OPTICAL_FLOW_USAGE_OUTPUT_BIT_NV, VK_FORMAT_R16G16_S10_5_NV); if (s->flow_format != VK_FORMAT_R16G16_S10_5_NV) { av_log(avctx, AV_LOG_ERROR, "Optical flow SFIXED5 vector format " "unavailable (got %d)\n", s->flow_format); return AVERROR(ENOTSUP); } static const struct { int size; VkOpticalFlowGridSizeFlagsNV bit; } grid_map[] = { { 1, VK_OPTICAL_FLOW_GRID_SIZE_1X1_BIT_NV }, { 2, VK_OPTICAL_FLOW_GRID_SIZE_2X2_BIT_NV }, { 4, VK_OPTICAL_FLOW_GRID_SIZE_4X4_BIT_NV }, { 8, VK_OPTICAL_FLOW_GRID_SIZE_8X8_BIT_NV }, }; grids = vkctx->optical_flow_props.supportedOutputGridSizes; if (s->opt_grid_size) { /* Honour an explicit grid request, erroring if the device lacks it. */ VkOpticalFlowGridSizeFlagsNV want = 0; for (int i = 0; i < FF_ARRAY_ELEMS(grid_map); i++) if (grid_map[i].size == s->opt_grid_size) want = grid_map[i].bit; if (!want || !(grids & want)) { av_log(avctx, AV_LOG_ERROR, "Requested optical flow grid size %d is not " "supported by the device (supported mask 0x%x)\n", s->opt_grid_size, grids); return AVERROR(ENOTSUP); } s->grid_size = s->opt_grid_size; s->grid_bit = want; } else { /* Auto: pick the finest (smallest) grid the device supports. */ s->grid_size = 0; for (int i = 0; i < FF_ARRAY_ELEMS(grid_map); i++) { if (grids & grid_map[i].bit) { s->grid_size = grid_map[i].size; s->grid_bit = grid_map[i].bit; break; } } if (!s->grid_size) { av_log(avctx, AV_LOG_ERROR, "No supported optical flow output grid size\n"); return AVERROR(ENOTSUP); } } s->flow_width = (s->width + s->grid_size - 1) / s->grid_size; s->flow_height = (s->height + s->grid_size - 1) / s->grid_size; av_log(avctx, AV_LOG_INFO, "optical flow: perf %d, grid %d, flow %dx%d, bidir=%d, " "min %dx%d max %dx%d\n", s->perf_level, s->grid_size, s->flow_width, s->flow_height, vkctx->optical_flow_props.bidirectionalFlowSupported, vkctx->optical_flow_props.minWidth, vkctx->optical_flow_props.minHeight, vkctx->optical_flow_props.maxWidth, vkctx->optical_flow_props.maxHeight); { const VkPhysicalDeviceOpticalFlowPropertiesNV *ofp = &vkctx->optical_flow_props; if ((uint32_t)s->width < ofp->minWidth || (uint32_t)s->width > ofp->maxWidth || (uint32_t)s->height < ofp->minHeight || (uint32_t)s->height > ofp->maxHeight) { av_log(avctx, AV_LOG_ERROR, "Frame size %dx%d is outside the range the " "device optical flow engine supports (%ux%u to %ux%u)\n", s->width, s->height, ofp->minWidth, ofp->minHeight, ofp->maxWidth, ofp->maxHeight); return AVERROR(ENOTSUP); } } if (!vkctx->optical_flow_props.bidirectionalFlowSupported) { av_log(avctx, AV_LOG_ERROR, "Device optical flow engine does not support " "bidirectional flow, which this filter requires\n"); return AVERROR(ENOTSUP); } /* Create the persistent optical flow images and sessions, one set per slot * so consecutive source pairs round-robin between independent resources. */ for (int slot = 0; slot < FRUC_NB_SLOTS; slot++) { FRUCFlowSlot *fs = &s->slots[slot]; RET(create_of_image(s, &fs->gray_img[0], &fs->gray_mem[0], &fs->gray_view[0], s->input_format, s->width, s->height, VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV, VK_IMAGE_USAGE_STORAGE_BIT, 0)); RET(create_of_image(s, &fs->gray_img[1], &fs->gray_mem[1], &fs->gray_view[1], s->input_format, s->width, s->height, VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV, VK_IMAGE_USAGE_STORAGE_BIT, 0)); /* The flow images must be created mutable, as we need to sample the raw * integers through an R16G16_SINT view. SFIXED5 advertises SAMPLED_IMAGE, * but if you try and use a float sampler, it will read the values as * R16G16_SFLOAT, resulting in garbage. So we have to read the raw bits and * rescale them (value/32) ourselves. */ RET(create_of_image(s, &fs->flow_img[0], &fs->flow_mem[0], &fs->flow_view[0], s->flow_format, s->flow_width, s->flow_height, VK_OPTICAL_FLOW_USAGE_OUTPUT_BIT_NV, VK_IMAGE_USAGE_SAMPLED_BIT, VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT)); RET(create_of_image(s, &fs->flow_img[1], &fs->flow_mem[1], &fs->flow_view[1], s->flow_format, s->flow_width, s->flow_height, VK_OPTICAL_FLOW_USAGE_OUTPUT_BIT_NV, VK_IMAGE_USAGE_SAMPLED_BIT, VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT)); for (int i = 0; i < 2; i++) { VkImageViewCreateInfo view_info = { .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = fs->flow_img[i], .viewType = VK_IMAGE_VIEW_TYPE_2D, .format = VK_FORMAT_R16G16_SINT, .components = ff_comp_identity_map, .subresourceRange = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .levelCount = 1, .layerCount = 1, }, }; if (vk->CreateImageView(vkctx->hwctx->act_dev, &view_info, vkctx->hwctx->alloc, &fs->flow_sint_view[i]) != VK_SUCCESS) { av_log(avctx, AV_LOG_ERROR, "Failed to create flow SINT view\n"); return AVERROR_EXTERNAL; } } ret = vk->CreateOpticalFlowSessionNV(vkctx->hwctx->act_dev, &(VkOpticalFlowSessionCreateInfoNV) { .sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_INFO_NV, .width = s->width, .height = s->height, .imageFormat = s->input_format, .flowVectorFormat = s->flow_format, .outputGridSize = s->grid_bit, .performanceLevel = s->perf_level, .flags = VK_OPTICAL_FLOW_SESSION_CREATE_BOTH_DIRECTIONS_BIT_NV, }, vkctx->hwctx->alloc, &fs->session); if (ret != VK_SUCCESS) { av_log(avctx, AV_LOG_ERROR, "Failed to create optical flow session: %s\n", ff_vk_ret2str(ret)); return AVERROR_EXTERNAL; } static const VkOpticalFlowSessionBindingPointNV binding_points[] = { VK_OPTICAL_FLOW_SESSION_BINDING_POINT_INPUT_NV, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_REFERENCE_NV, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_FLOW_VECTOR_NV, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_BACKWARD_FLOW_VECTOR_NV, }; const VkImageView binding_views[] = { fs->gray_view[0], fs->gray_view[1], fs->flow_view[0], fs->flow_view[1], }; for (int i = 0; i < FF_ARRAY_ELEMS(binding_points); i++) { ret = vk->BindOpticalFlowSessionImageNV(vkctx->hwctx->act_dev, fs->session, binding_points[i], binding_views[i], VK_IMAGE_LAYOUT_GENERAL); if (ret != VK_SUCCESS) { av_log(avctx, AV_LOG_ERROR, "Failed to bind optical flow session " "image: %s\n", ff_vk_ret2str(ret)); return AVERROR_EXTERNAL; } } } RET(init_image_layouts(s)); /* Grayscale extraction shader. */ ff_vk_shader_load(&s->grayscale, VK_SHADER_STAGE_COMPUTE_BIT, NULL, (uint32_t []) { 32, 32, 1 }, 0); ff_vk_shader_add_push_const(&s->grayscale, 0, sizeof(GrayscalePushData), VK_SHADER_STAGE_COMPUTE_BIT); { const FFVulkanDescriptorSetBinding desc[] = { { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .dimensions = 2, .elems = s->gray_planes, .stages = VK_SHADER_STAGE_COMPUTE_BIT, .samplers = DUP_SAMPLER(s->sampler), }, { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .dimensions = 2, .elems = s->gray_planes, .stages = VK_SHADER_STAGE_COMPUTE_BIT, .samplers = DUP_SAMPLER(s->sampler), }, { .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .mem_layout = "r8", .mem_quali = "writeonly", .dimensions = 2, .elems = 2, .stages = VK_SHADER_STAGE_COMPUTE_BIT, }, }; ff_vk_shader_add_descriptor_set(vkctx, &s->grayscale, desc, 3, 0); } RET(ff_vk_shader_link(vkctx, &s->grayscale, ff_fruc_grayscale_comp_spv_data, ff_fruc_grayscale_comp_spv_len, "main")); RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->grayscale)); /* Motion compensated interpolation shader. */ ff_vk_shader_load(&s->interpolate, VK_SHADER_STAGE_COMPUTE_BIT, NULL, (uint32_t []) { 32, 32, 1 }, 0); ff_vk_shader_add_push_const(&s->interpolate, 0, sizeof(InterpolatePushData), VK_SHADER_STAGE_COMPUTE_BIT); { const FFVulkanDescriptorSetBinding desc[] = { { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .dimensions = 2, .elems = planes, .stages = VK_SHADER_STAGE_COMPUTE_BIT, .samplers = DUP_SAMPLER(s->sampler), }, { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .dimensions = 2, .elems = planes, .stages = VK_SHADER_STAGE_COMPUTE_BIT, .samplers = DUP_SAMPLER(s->sampler), }, { .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .mem_quali = "writeonly", .dimensions = 2, .elems = planes, .stages = VK_SHADER_STAGE_COMPUTE_BIT, }, { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .dimensions = 2, .stages = VK_SHADER_STAGE_COMPUTE_BIT, .samplers = DUP_SAMPLER(s->flow_sampler), }, { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .dimensions = 2, .stages = VK_SHADER_STAGE_COMPUTE_BIT, .samplers = DUP_SAMPLER(s->flow_sampler), }, }; ff_vk_shader_add_descriptor_set(vkctx, &s->interpolate, desc, 5, 0); } RET(ff_vk_shader_link(vkctx, &s->interpolate, ff_fruc_interpolate_comp_spv_data, ff_fruc_interpolate_comp_spv_len, "main")); RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->interpolate)); fail: return err; } static void of_image_barrier(VkImageMemoryBarrier2 *bar, VkImage img, VkPipelineStageFlags2 src_stage, VkAccessFlags2 src_access, VkPipelineStageFlags2 dst_stage, VkAccessFlags2 dst_access) { *bar = (VkImageMemoryBarrier2) { .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = src_stage, .srcAccessMask = src_access, .dstStageMask = dst_stage, .dstAccessMask = dst_access, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = img, .subresourceRange = { .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .levelCount = 1, .layerCount = 1, }, }; } /* Compute the forward and backward optical flow between f0 and f1. */ static int compute_flow(AVFilterContext *avctx) { int err; FRUCVulkanContext *s = avctx->priv; FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; FFVkExecContext *exec; VkImageView f0_views[AV_NUM_DATA_POINTERS]; VkImageView f1_views[AV_NUM_DATA_POINTERS]; /* f0 and f1 each contribute one barrier per VkImage (a multi-image * sw_format such as planar RGB or a separate alpha plane has several), * plus the two single-image grayscale targets. */ VkImageMemoryBarrier2 img_bar[2 * AV_NUM_DATA_POINTERS + 2]; int nb_img_bar; /* This pair's generation; sem_gray and sem_flow are signalled with it. */ s->gen++; /* Round-robin slot for this pair's optical flow resources. */ FRUCFlowSlot *fs = &s->slots[s->gen % FRUC_NB_SLOTS]; /* --- Grayscale extraction on the compute queue. --- */ exec = ff_vk_exec_get(vkctx, &s->e); err = ff_vk_exec_start(vkctx, exec); if (err < 0) return err; RET(ff_vk_exec_add_dep_frame(vkctx, exec, s->f0, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); RET(ff_vk_exec_add_dep_frame(vkctx, exec, s->f1, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); /* Wait for the prior occupant of this slot (FRUC_NB_SLOTS pairs ago) to * finish reading its grayscale images on the flow engine before overwriting * them; the slot's first uses have no prior occupant. */ if (s->gen > FRUC_NB_SLOTS) ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_flow, s->gen - FRUC_NB_SLOTS, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); /* Serialize the sem_gray signals: the exec pool may round-robin consecutive * generations onto different queues, which have no implicit ordering, so * wait for the previous generation's signal before emitting this one. * Otherwise generation N+1 could signal the smaller value N+1 before N, * which is invalid for a timeline semaphore. Value 0 is the initial state. */ ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_gray, s->gen - 1, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); ff_vk_exec_add_dep_signal_sem(vkctx, exec, s->sem_gray, s->gen, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); RET(ff_vk_create_imageviews(vkctx, exec, f0_views, s->f0, FF_VK_REP_FLOAT)); RET(ff_vk_create_imageviews(vkctx, exec, f1_views, s->f1, FF_VK_REP_FLOAT)); for (int p = 0; p < s->gray_planes; p++) { ff_vk_shader_update_img(vkctx, exec, &s->grayscale, 0, 0, p, f0_views[p], VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, s->sampler); ff_vk_shader_update_img(vkctx, exec, &s->grayscale, 0, 1, p, f1_views[p], VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, s->sampler); } ff_vk_shader_update_img(vkctx, exec, &s->grayscale, 0, 2, 0, fs->gray_view[0], VK_IMAGE_LAYOUT_GENERAL, VK_NULL_HANDLE); ff_vk_shader_update_img(vkctx, exec, &s->grayscale, 0, 2, 1, fs->gray_view[1], VK_IMAGE_LAYOUT_GENERAL, VK_NULL_HANDLE); ff_vk_exec_bind_shader(vkctx, exec, &s->grayscale); { GrayscalePushData pd; memcpy(pd.luma_weights, s->luma_weights, sizeof(pd.luma_weights)); pd.planes = s->gray_planes; ff_vk_shader_update_push_const(vkctx, exec, &s->grayscale, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(GrayscalePushData), &pd); } nb_img_bar = 0; ff_vk_frame_barrier(vkctx, exec, s->f0, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_QUEUE_FAMILY_IGNORED); ff_vk_frame_barrier(vkctx, exec, s->f1, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_QUEUE_FAMILY_IGNORED); of_image_barrier(&img_bar[nb_img_bar++], fs->gray_img[0], VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 0, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT); of_image_barrier(&img_bar[nb_img_bar++], fs->gray_img[1], VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 0, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT); vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .pImageMemoryBarriers = img_bar, .imageMemoryBarrierCount = nb_img_bar, }); vk->CmdDispatch(exec->buf, FFALIGN(s->width, s->grayscale.lg_size[0]) / s->grayscale.lg_size[0], FFALIGN(s->height, s->grayscale.lg_size[1]) / s->grayscale.lg_size[1], 1); /* sem_gray orders the optical flow submission after this one * and makes the grayscale writes visible across the queue boundary. */ RET(ff_vk_exec_submit(vkctx, exec)); /* --- Optical flow execution on the optical flow queue. --- */ exec = ff_vk_exec_get(vkctx, &s->e_of); err = ff_vk_exec_start(vkctx, exec); if (err < 0) return err; /* Wait for the grayscale writes, signal once the flow has been written. */ ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_gray, s->gen, VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV); /* Wait for the slot's prior occupant to finish sampling its flow images * (fs->interp_done) before overwriting them; 0 is the initial state of an * unused slot and is satisfied immediately. */ ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_interp, fs->interp_done, VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV); /* Serialize the sem_flow signals for the same reason as sem_gray above: the * optical flow contexts may span multiple queues, so wait for the previous * generation's flow signal before signaling this one to keep the timeline * values monotonic. Value 0 is the initial state. */ ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_flow, s->gen - 1, VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV); ff_vk_exec_add_dep_signal_sem(vkctx, exec, s->sem_flow, s->gen, VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV); /* The flow images stay in VK_IMAGE_LAYOUT_GENERAL and the semaphores handle * cross-queue visibility, so no image barriers are needed here. */ vk->CmdOpticalFlowExecuteNV(exec->buf, fs->session, &(VkOpticalFlowExecuteInfoNV) { .sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_EXECUTE_INFO_NV, }); /* sem_flow orders the interpolation after the flow execution * and makes the flow writes visible on the compute queue. */ RET(ff_vk_exec_submit(vkctx, exec)); s->flow_valid = 1; return 0; fail: ff_vk_exec_discard(vkctx, exec); return err; } /* Visible texel extent of a plane: plane 0 (and non-planar / alpha) is full size, * chroma planes are subsampled. Mirrors hwcontext_vulkan's get_plane_wh, which is * how the frames context sizes each plane's image. */ static void plane_wh(const AVPixFmtDescriptor *desc, int width, int height, int plane, uint32_t *w, uint32_t *h) { int sub = plane && plane != 3 && (desc->flags & AV_PIX_FMT_FLAG_PLANAR) && !(desc->flags & AV_PIX_FMT_FLAG_RGB); *w = sub ? AV_CEIL_RSHIFT(width, desc->log2_chroma_w) : width; *h = sub ? AV_CEIL_RSHIFT(height, desc->log2_chroma_h) : height; } /* Produce the motion compensated output frame at temporal position t. */ static int interpolate_frame(AVFilterContext *avctx, AVFrame *out, float t) { int err; FRUCVulkanContext *s = avctx->priv; FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; FFVkExecContext *exec; VkImageView f0_views[AV_NUM_DATA_POINTERS]; VkImageView f1_views[AV_NUM_DATA_POINTERS]; VkImageView out_views[AV_NUM_DATA_POINTERS]; /* out, f0 and f1 each contribute one barrier per VkImage (a multi-image * sw_format such as planar RGB or a separate alpha plane has several), * plus the two single-image flow fields. */ VkImageMemoryBarrier2 img_bar[3 * AV_NUM_DATA_POINTERS + 2]; int nb_img_bar; const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(vkctx->output_format); InterpolatePushData pd = { .t = t, .planes = av_pix_fmt_count_planes(vkctx->output_format), }; memcpy(pd.luma_weights, s->luma_weights, sizeof(pd.luma_weights)); /* The shader works in the visible frame's coordinate space; the source and * output images may each be allocated larger than that. */ for (int i = 0; i < pd.planes; i++) { uint32_t w, h; plane_wh(desc, s->width, s->height, i, &w, &h); pd.plane_size[i][0] = w; pd.plane_size[i][1] = h; } /* Not via RET: the fail label discards deps on exec, which is not yet * acquired here, and compute_flow cleans up its own exec on failure. */ if (!s->flow_valid) { err = compute_flow(avctx); if (err < 0) return err; } /* Same slot compute_flow selected for this pair's generation. */ FRUCFlowSlot *fs = &s->slots[s->gen % FRUC_NB_SLOTS]; exec = ff_vk_exec_get(vkctx, &s->e); err = ff_vk_exec_start(vkctx, exec); if (err < 0) return err; /* Every interpolation of this pair waits on the same flow result, keyed by * the pair generation. */ ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_flow, s->gen, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); /* Chain sem_interp: wait the previous value, signal the next. Keeps the * signals monotonic and lets the next pair's optical flow fence on the final * value before overwriting the flow images. */ ff_vk_exec_add_dep_wait_sem(vkctx, exec, s->sem_interp, s->interp_value, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); ff_vk_exec_add_dep_signal_sem(vkctx, exec, s->sem_interp, ++s->interp_value, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); /* Record this pair's value so the next occupant of the slot can fence on it. */ fs->interp_done = s->interp_value; RET(ff_vk_exec_add_dep_frame(vkctx, exec, out, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); RET(ff_vk_exec_add_dep_frame(vkctx, exec, s->f0, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); RET(ff_vk_exec_add_dep_frame(vkctx, exec, s->f1, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); RET(ff_vk_create_imageviews(vkctx, exec, f0_views, s->f0, FF_VK_REP_FLOAT)); RET(ff_vk_create_imageviews(vkctx, exec, f1_views, s->f1, FF_VK_REP_FLOAT)); RET(ff_vk_create_imageviews(vkctx, exec, out_views, out, FF_VK_REP_FLOAT)); ff_vk_shader_update_img_array(vkctx, exec, &s->interpolate, s->f0, f0_views, 0, 0, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, s->sampler); ff_vk_shader_update_img_array(vkctx, exec, &s->interpolate, s->f1, f1_views, 0, 1, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, s->sampler); ff_vk_shader_update_img_array(vkctx, exec, &s->interpolate, out, out_views, 0, 2, VK_IMAGE_LAYOUT_GENERAL, VK_NULL_HANDLE); ff_vk_shader_update_img(vkctx, exec, &s->interpolate, 0, 3, 0, fs->flow_sint_view[0], VK_IMAGE_LAYOUT_GENERAL, s->flow_sampler); ff_vk_shader_update_img(vkctx, exec, &s->interpolate, 0, 4, 0, fs->flow_sint_view[1], VK_IMAGE_LAYOUT_GENERAL, s->flow_sampler); ff_vk_exec_bind_shader(vkctx, exec, &s->interpolate); ff_vk_shader_update_push_const(vkctx, exec, &s->interpolate, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pd), &pd); nb_img_bar = 0; ff_vk_frame_barrier(vkctx, exec, out, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_IMAGE_LAYOUT_GENERAL, VK_QUEUE_FAMILY_IGNORED); ff_vk_frame_barrier(vkctx, exec, s->f0, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_QUEUE_FAMILY_IGNORED); ff_vk_frame_barrier(vkctx, exec, s->f1, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_QUEUE_FAMILY_IGNORED); of_image_barrier(&img_bar[nb_img_bar++], fs->flow_img[0], VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 0, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT); of_image_barrier(&img_bar[nb_img_bar++], fs->flow_img[1], VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 0, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_READ_BIT); vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .pImageMemoryBarriers = img_bar, .imageMemoryBarrierCount = nb_img_bar, }); vk->CmdDispatch(exec->buf, FFALIGN(s->width, s->interpolate.lg_size[0]) / s->interpolate.lg_size[0], FFALIGN(s->height, s->interpolate.lg_size[1]) / s->interpolate.lg_size[1], 1); return ff_vk_exec_submit(vkctx, exec); fail: ff_vk_exec_discard(vkctx, exec); return err; } static int copy_frame(AVFilterContext *avctx, AVFrame *out, AVFrame *src) { int err; FRUCVulkanContext *s = avctx->priv; FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; FFVkExecContext *exec; AVVkFrame *src_vk = (AVVkFrame *)src->data[0]; AVVkFrame *out_vk = (AVVkFrame *)out->data[0]; const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(vkctx->output_format); const int nb_planes = av_pix_fmt_count_planes(vkctx->output_format); const int src_nb_images = ff_vk_count_images(src_vk); const int out_nb_images = ff_vk_count_images(out_vk); /* src and out each contribute one barrier per VkImage; a multi-image * sw_format such as planar RGB or a separate alpha plane has several. */ VkImageMemoryBarrier2 img_bar[2 * AV_NUM_DATA_POINTERS]; int nb_img_bar = 0; exec = ff_vk_exec_get(vkctx, &s->e); err = ff_vk_exec_start(vkctx, exec); if (err < 0) return err; RET(ff_vk_exec_add_dep_frame(vkctx, exec, src, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT)); RET(ff_vk_exec_add_dep_frame(vkctx, exec, out, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT)); ff_vk_frame_barrier(vkctx, exec, src, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_QUEUE_FAMILY_IGNORED); ff_vk_frame_barrier(vkctx, exec, out, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_QUEUE_FAMILY_IGNORED); vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .pImageMemoryBarriers = img_bar, .imageMemoryBarrierCount = nb_img_bar, }); for (int i = 0; i < nb_planes; i++) { uint32_t w, h; plane_wh(desc, s->width, s->height, i, &w, &h); VkImageCopy region = { .srcSubresource = { .aspectMask = ff_vk_aspect_flag(src, i), .layerCount = 1 }, .dstSubresource = { .aspectMask = ff_vk_aspect_flag(out, i), .layerCount = 1 }, .extent = { w, h, 1 }, }; vk->CmdCopyImage(exec->buf, src_vk->img[FFMIN(i, src_nb_images - 1)], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, out_vk->img[FFMIN(i, out_nb_images - 1)], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); } return ff_vk_exec_submit(vkctx, exec); fail: ff_vk_exec_discard(vkctx, exec); return err; } /* Emit src unchanged. The frame must be copied rather than cloned: a clone keeps * the input's frames context, and the frame has to match the output context the * downstream link is configured for. */ static int passthrough_frame(AVFilterContext *ctx, AVFrame **work, AVFrame *src) { AVFilterLink *outlink = ctx->outputs[0]; int ret; *work = ff_get_video_buffer(outlink, outlink->w, outlink->h); if (!*work) return AVERROR(ENOMEM); ret = av_frame_copy_props(*work, src); if (ret < 0) goto fail; ret = copy_frame(ctx, *work, src); if (ret < 0) goto fail; return 0; fail: av_frame_free(work); return ret; } static int process_work_frame(AVFilterContext *ctx) { FRUCVulkanContext *s = ctx->priv; AVFilterLink *outlink = ctx->outputs[0]; int64_t work_pts; int64_t interpolate8; int ret; if (!s->f1) return 0; if (!s->f0 && !s->flush) return 0; work_pts = s->start_pts + av_rescale_q(s->n, av_inv_q(s->dest_frame_rate), s->dest_time_base); if (work_pts >= s->pts1 && !s->flush) return 0; if (!s->f0) { av_assert1(s->flush); ret = passthrough_frame(ctx, &s->work, s->f1); if (ret < 0) return ret; /* We are flushing, so f1 is not needed once passed through. Free it so * the flush terminates instead of re-emitting it every activation. */ av_frame_free(&s->f1); } else { if (work_pts >= s->pts1 + s->delta && s->flush) return 0; interpolate8 = av_rescale(work_pts - s->pts0, 256, s->delta); if (interpolate8 >= 256) { ret = passthrough_frame(ctx, &s->work, s->f1); if (ret < 0) return ret; } else if (interpolate8 <= 0) { ret = passthrough_frame(ctx, &s->work, s->f0); if (ret < 0) return ret; } else { float t = (float)(work_pts - s->pts0) / (float)s->delta; s->work = ff_get_video_buffer(outlink, outlink->w, outlink->h); if (!s->work) return AVERROR(ENOMEM); ret = av_frame_copy_props(s->work, s->f0); if (ret < 0) { av_frame_free(&s->work); return ret; } ret = interpolate_frame(ctx, s->work, t); if (ret < 0) { av_frame_free(&s->work); return ret; } } } s->work->pts = work_pts; s->n++; return 1; } static int activate(AVFilterContext *ctx) { int ret, status; AVFilterLink *inlink = ctx->inputs[0]; AVFilterLink *outlink = ctx->outputs[0]; FRUCVulkanContext *s = ctx->priv; AVFrame *inpicref; int64_t pts; FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); retry: ret = process_work_frame(ctx); if (ret < 0) return ret; else if (ret == 1) return ff_filter_frame(outlink, s->work); ret = ff_inlink_consume_frame(inlink, &inpicref); if (ret < 0) return ret; if (inpicref) { if (inpicref->flags & AV_FRAME_FLAG_INTERLACED) av_log(ctx, AV_LOG_WARNING, "Interlaced frame found - the output will not be correct.\n"); if (inpicref->pts == AV_NOPTS_VALUE) { av_log(ctx, AV_LOG_WARNING, "Ignoring frame without PTS.\n"); av_frame_free(&inpicref); } } if (inpicref) { pts = av_rescale_q(inpicref->pts, s->srce_time_base, s->dest_time_base); if (s->f1 && pts == s->pts1) { av_log(ctx, AV_LOG_WARNING, "Ignoring frame with same PTS.\n"); av_frame_free(&inpicref); } } if (inpicref) { av_frame_free(&s->f0); s->f0 = s->f1; s->pts0 = s->pts1; s->f1 = inpicref; s->pts1 = pts; s->delta = s->pts1 - s->pts0; s->flow_valid = 0; if (s->delta < 0) { av_log(ctx, AV_LOG_WARNING, "PTS discontinuity.\n"); s->start_pts = s->pts1; s->n = 0; av_frame_free(&s->f0); } if (s->start_pts == AV_NOPTS_VALUE) s->start_pts = s->pts1; goto retry; } if (ff_inlink_acknowledge_status(inlink, &status, &pts)) { if (!s->flush) { s->flush = 1; goto retry; } ff_outlink_set_status(outlink, status, pts); return 0; } FF_FILTER_FORWARD_WANTED(outlink, inlink); return FFERROR_NOT_READY; } static int config_input(AVFilterLink *inlink) { AVFilterContext *ctx = inlink->dst; FRUCVulkanContext *s = ctx->priv; s->srce_time_base = inlink->time_base; return ff_vk_filter_config_input(inlink); } static int config_output(AVFilterLink *outlink) { AVFilterContext *ctx = outlink->src; AVFilterLink *inlink = ctx->inputs[0]; FilterLink *il = ff_filter_link(inlink); FilterLink *ol = ff_filter_link(outlink); FRUCVulkanContext *s = ctx->priv; double var_values[VARS_NB], res; int err; int exact; ff_dlog(ctx, "config_output()\n"); ff_dlog(ctx, "config_output() input time base:%u/%u (%f)\n", ctx->inputs[0]->time_base.num,ctx->inputs[0]->time_base.den, av_q2d(ctx->inputs[0]->time_base)); // The fps option is an expression evaluated against the source frame rate var_values[VAR_SOURCE_FPS] = av_q2d(il->frame_rate); err = av_expr_parse_and_eval(&res, s->requested_frame_rate, var_names, var_values, NULL, NULL, NULL, NULL, NULL, 0, ctx); if (err < 0) return err; s->dest_frame_rate = av_d2q(res, INT_MAX); if (s->dest_frame_rate.num <= 0 || s->dest_frame_rate.den <= 0) { av_log(ctx, AV_LOG_ERROR, "Invalid output frame rate '%s' (must evaluate to a positive value)\n", s->requested_frame_rate); return AVERROR(EINVAL); } // make sure timebase is small enough to hold the framerate exact = av_reduce(&s->dest_time_base.num, &s->dest_time_base.den, av_gcd((int64_t)s->srce_time_base.num * s->dest_frame_rate.num, (int64_t)s->srce_time_base.den * s->dest_frame_rate.den ), (int64_t)s->srce_time_base.den * s->dest_frame_rate.num, INT_MAX); /* The source-timebase-derived reduction above can collapse to a zero time * base (av_reduce() bounds its result, so the numerator can underflow to * zero, leaving 0/1) when the source timebase shares no useful factors with * the requested rate, which would make the output timebase unusable. Fall * back to the plain 1/fps timebase in that case so a valid timebase is * always produced. */ if (!s->dest_time_base.num || !s->dest_time_base.den) { exact = av_reduce(&s->dest_time_base.num, &s->dest_time_base.den, s->dest_frame_rate.den, s->dest_frame_rate.num, INT_MAX); } av_log(ctx, AV_LOG_INFO, "time base:%u/%u -> %u/%u exact:%d\n", s->srce_time_base.num, s->srce_time_base.den, s->dest_time_base.num, s->dest_time_base.den, exact); if (!exact) { av_log(ctx, AV_LOG_WARNING, "Timebase conversion is not exact\n"); } err = ff_vk_filter_config_output(outlink); if (err < 0) return err; ol->frame_rate = s->dest_frame_rate; outlink->time_base = s->dest_time_base; ff_dlog(ctx, "config_output() output time base:%u/%u (%f) w:%d h:%d\n", outlink->time_base.num, outlink->time_base.den, av_q2d(outlink->time_base), outlink->w, outlink->h); return init_filter(ctx); } static av_cold int init(AVFilterContext *avctx) { FRUCVulkanContext *s = avctx->priv; s->start_pts = AV_NOPTS_VALUE; return ff_vk_filter_init(avctx); } static av_cold void uninit(AVFilterContext *avctx) { FRUCVulkanContext *s = avctx->priv; FFVulkanContext *vkctx = &s->vkctx; FFVulkanFunctions *vk = &vkctx->vkfn; /* Free the execution pools first: this waits for every submitted command * buffer to retire (ff_vk_exec_pool_free fences each context). The pooled * submissions wait on and signal the timeline semaphores and reference the * optical flow images below, so those objects must outlive the wait — destroying * an in-use semaphore would leave a submission's fence permanently unsignaled * and deadlock the wait. */ ff_vk_exec_pool_free(vkctx, &s->e); ff_vk_exec_pool_free(vkctx, &s->e_of); if (s->sem_gray) vk->DestroySemaphore(vkctx->hwctx->act_dev, s->sem_gray, vkctx->hwctx->alloc); if (s->sem_flow) vk->DestroySemaphore(vkctx->hwctx->act_dev, s->sem_flow, vkctx->hwctx->alloc); if (s->sem_interp) vk->DestroySemaphore(vkctx->hwctx->act_dev, s->sem_interp, vkctx->hwctx->alloc); for (int slot = 0; slot < FRUC_NB_SLOTS; slot++) { FRUCFlowSlot *fs = &s->slots[slot]; if (fs->session) vk->DestroyOpticalFlowSessionNV(vkctx->hwctx->act_dev, fs->session, vkctx->hwctx->alloc); for (int i = 0; i < 2; i++) { if (fs->gray_view[i]) vk->DestroyImageView(vkctx->hwctx->act_dev, fs->gray_view[i], vkctx->hwctx->alloc); ff_vk_image_free(vkctx, &fs->gray_img[i], &fs->gray_mem[i]); if (fs->flow_view[i]) vk->DestroyImageView(vkctx->hwctx->act_dev, fs->flow_view[i], vkctx->hwctx->alloc); if (fs->flow_sint_view[i]) vk->DestroyImageView(vkctx->hwctx->act_dev, fs->flow_sint_view[i], vkctx->hwctx->alloc); ff_vk_image_free(vkctx, &fs->flow_img[i], &fs->flow_mem[i]); } } ff_vk_shader_free(vkctx, &s->grayscale); ff_vk_shader_free(vkctx, &s->interpolate); if (s->sampler) vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler, vkctx->hwctx->alloc); if (s->flow_sampler) vk->DestroySampler(vkctx->hwctx->act_dev, s->flow_sampler, vkctx->hwctx->alloc); ff_vk_uninit(vkctx); av_frame_free(&s->f0); av_frame_free(&s->f1); } static const AVFilterPad fruc_vulkan_inputs[] = { { .name = "default", .type = AVMEDIA_TYPE_VIDEO, .config_props = config_input, }, }; static const AVFilterPad fruc_vulkan_outputs[] = { { .name = "default", .type = AVMEDIA_TYPE_VIDEO, .config_props = config_output, }, }; const FFFilter ff_vf_fruc_vulkan = { .p.name = "fruc_vulkan", .p.description = NULL_IF_CONFIG_SMALL("Frame rate up-conversion using the Vulkan NV optical flow extension"), .p.priv_class = &fruc_vulkan_class, .p.flags = AVFILTER_FLAG_HWDEVICE, .priv_size = sizeof(FRUCVulkanContext), .init = init, .uninit = uninit, FILTER_INPUTS(fruc_vulkan_inputs), FILTER_OUTPUTS(fruc_vulkan_outputs), FILTER_SINGLE_PIXFMT(AV_PIX_FMT_VULKAN), .activate = activate, .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, };