Initial work on vulkan video enc

.github/workflows/rust.yml

@@ -17,12 +17,12 @@ jobs:
       image: archlinux:latest
     steps:
     - name: Install system deps
-      run: pacman  --noconfirm -Syu ffmpeg rustup gcc clang pkgconf
+      run: pacman  --noconfirm -Syu ffmpeg rustup gcc clang pkgconf vulkan-headers
     - name: Install rust
       run: rustup install stable
     - uses: actions/checkout@v4
     - name: Build
-      run: cargo build --verbose
+      run: cargo build --verbose --locked --all-features
     - name: Format
       run: cargo fmt --check
     #- name: Run tests

.gitignore

@@ -1,3 +1,4 @@
 /target
+/.vscode
 *.avi
 *.mp4

.vscode/launch.json

@@ -22,6 +22,42 @@
             "args": [],
             "cwd": "${workspaceFolder}"
         },
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug executable 'wl-screenrec' (WAYLAND-2)",
+            "cargo": {
+                "args": [
+                    "build",
+                    "--bin=wl-screenrec",
+                    "--package=wl-screenrec",
+                    "--features=experimental-vulkan"
+                ],
+                "filter": {
+                    "name": "wl-screenrec",
+                    "kind": "bin"
+                },
+                "env": {
+                    "FFMPEG_DIR": "/home/russell/ffmpeg/inst"
+                }
+            },
+            "env": {
+                "WAYLAND_DISPLAY": "wayland-2",
+                "RUST_BACKTRACE": "full",
+                "LD_LIBRARY_PATH": "/home/russell/ffmpeg/inst/lib",
+                "VK_ICD_FILENAMES": "/home/russell/mesa/build/src/intel/vulkan/intel_devenv_icd.x86_64.json"
+            },
+            "preRunCommands": [
+                "process handle -p true -s false INT"
+            ],
+            "args": [
+                "--experimental-ext-image-copy-capture", 
+                "--experimental-vulkan",
+                "--codec=avc", 
+                "-vvv",
+            ],
+            "cwd": "${workspaceFolder}"
+        },
         {
             "type": "lldb",
             "request": "launch",

Cargo.toml

@@ -28,7 +28,7 @@ wayland-protocols = { version = "0.32", features = [
 ] }
 wayland-protocols-wlr = { version = "0.3", features = ["client"] }
 ffmpeg-next = "7.0.1"
-ffmpeg-sys-next = "7.0.0" # need direct dep on -sys to get metadata to consume in build.rs
+ffmpeg-sys-next = { version = "7.0.0" }
 thiserror = "2.0.3"
 human-size = "0.4.2"
 signal-hook = "0.3.15"
@@ -39,6 +39,7 @@ log = "0.4.21"
 clap_complete = "4.5.8"
 log-once = "0.4.1"
 drm = "0.14.0"
+ash = { version = "0.38.0", optional = true }
 
 # [patch.crates-io]
 # ffmpeg-next = { path = "../rust-ffmpeg" }
@@ -52,5 +53,12 @@ nix = { version = "0.29.0", default-features = false, features = [
 ] }
 serde_json = "1.0.103"
 
+[features]
+experimental-vulkan = [
+    "ffmpeg-sys-next/vulkan", 
+    "ash"
+]
+
+
 [profile.release]
 lto = "thin"

build.rs

@@ -2,8 +2,9 @@ use std::env;
 
 fn main() {
     println!("cargo::rustc-check-cfg=cfg(ffmpeg_7_0)");
-    for (name, _value) in env::vars() {
-        if name.starts_with("DEP_FFMPEG_") {
+
+    for (name, value) in env::vars() {
+        if name.starts_with("DEP_FFMPEG_") && !value.is_empty() {
             println!(
                 r#"cargo::rustc-cfg={}"#,
                 name["DEP_FFMPEG_".len()..name.len()].to_lowercase()

src/avhw.rs

@@ -3,18 +3,19 @@ use std::{ffi::CString, path::Path, ptr::null_mut};
 use ffmpeg::{
     dict,
     ffi::{
-        av_buffer_ref, av_buffer_unref, av_hwdevice_ctx_create, av_hwframe_ctx_alloc,
-        av_hwframe_ctx_init, av_hwframe_get_buffer, AVHWFramesContext,
+        av_buffer_ref, av_buffer_unref, av_hwdevice_ctx_create, av_hwdevice_ctx_create_derived,
+        av_hwframe_ctx_alloc, av_hwframe_ctx_init, av_hwframe_get_buffer, AVHWFramesContext,
     },
     format::Pixel,
     frame,
 };
-use log::error;
 
 use crate::DrmModifier;
+use log::error;
 
 pub struct AvHwDevCtx {
     ptr: *mut ffmpeg::sys::AVBufferRef,
+    fmt: Pixel,
 }
 
 impl AvHwDevCtx {
@@ -38,7 +39,48 @@ impl AvHwDevCtx {
             if sts != 0 {
                 Err(ffmpeg::Error::from(sts))
             } else {
-                Ok(Self { ptr: hw_device_ctx })
+                Ok(Self {
+                    ptr: hw_device_ctx,
+                    fmt: Pixel::VAAPI,
+                })
+            }
+        }
+    }
+
+    pub fn new_vulkan(dri_device: &Path) -> Result<Self, ffmpeg::Error> {
+        unsafe {
+            let mut hw_device_ctx_drm = null_mut();
+            let mut hw_device_ctx = null_mut();
+
+            let dev_cstr = CString::new(dri_device.to_str().unwrap()).unwrap();
+
+            let sts = av_hwdevice_ctx_create(
+                &mut hw_device_ctx_drm,
+                ffmpeg_sys_next::AVHWDeviceType::AV_HWDEVICE_TYPE_DRM,
+                dev_cstr.as_ptr(),
+                null_mut(),
+                0,
+            );
+            if sts != 0 {
+                return Err(ffmpeg::Error::from(sts));
+            }
+
+            let sts = av_hwdevice_ctx_create_derived(
+                &mut hw_device_ctx,
+                ffmpeg_next::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN,
+                hw_device_ctx_drm,
+                0,
+            );
+
+            av_buffer_unref(&mut hw_device_ctx_drm);
+
+            if sts != 0 {
+                Err(ffmpeg::Error::from(sts))
+            } else {
+                Ok(Self {
+                    ptr: hw_device_ctx,
+                    fmt: Pixel::VULKAN,
+                })
             }
         }
     }
@@ -48,24 +90,88 @@ impl AvHwDevCtx {
         pixfmt: Pixel,
         width: i32,
         height: i32,
-        modifier: DrmModifier,
+        modifiers: &[DrmModifier],
     ) -> Result<AvHwFrameCtx, ffmpeg::Error> {
         unsafe {
             let mut hwframe = av_hwframe_ctx_alloc(self.ptr as *mut _);
-            let hwframe_casted = (*hwframe).data as *mut AVHWFramesContext;
+            let hwframe_casted = &mut *((*hwframe).data as *mut AVHWFramesContext);
 
             // ffmpeg does not expose RGB vaapi
-            (*hwframe_casted).format = Pixel::VAAPI.into();
-            (*hwframe_casted).sw_format = pixfmt.into();
-            (*hwframe_casted).width = width;
-            (*hwframe_casted).height = height;
-            (*hwframe_casted).initial_pool_size = 5;
-
-            if modifier != DrmModifier::LINEAR {
-                error!("unknown how to request non-linear frames in vaapi");
+            hwframe_casted.format = self.fmt.into();
+            hwframe_casted.sw_format = pixfmt.into();
+            hwframe_casted.width = width;
+            hwframe_casted.height = height;
+            hwframe_casted.initial_pool_size = 5;
+
+            let mut sts = -1;
+            if self.fmt == Pixel::VULKAN {
+                #[cfg(feature = "experimental-vulkan")]
+                {
+                    use ash::vk;
+                    use ffmpeg::ffi::{
+                        av_vkfmt_from_pixfmt, AVHWDeviceContext, AVVulkanDeviceContext,
+                        AVVulkanFramesContext,
+                    };
+
+                    let av_devctx = &(*((*self.as_mut_ptr()).data as *mut AVHWDeviceContext));
+                    let vk_hwctx = &*(av_devctx.hwctx as *mut AVVulkanDeviceContext);
+
+                    let inst = ash::Instance::load(
+                        &ash::StaticFn {
+                            get_instance_proc_addr: vk_hwctx.get_proc_addr,
+                        },
+                        vk_hwctx.inst,
+                    );
+
+                    let mut modifiers_filtered: Vec<DrmModifier> = Vec::new();
+                    for modifier in modifiers {
+                        let mut drm_info = ash::vk::PhysicalDeviceImageDrmFormatModifierInfoEXT {
+                            drm_format_modifier: modifier.0,
+
+                            ..Default::default()
+                        };
+                        let mut image_format_prop = ash::vk::ImageFormatProperties2 {
+                            ..Default::default()
+                        };
+
+                        if let Ok(()) = inst.get_physical_device_image_format_properties2(
+                            vk_hwctx.phys_dev,
+                            &vk::PhysicalDeviceImageFormatInfo2 {
+                                format: *av_vkfmt_from_pixfmt(pixfmt.into()),
+                                p_next: &mut drm_info as *mut _ as _,
+                                ..Default::default()
+                            },
+                            &mut image_format_prop,
+                        ) {
+                            modifiers_filtered.push(*modifier);
+                        }
+                    }
+
+                    // some buffer requirements are complex, just start removing modifiers until it works
+                    while sts != 0 && !modifiers_filtered.is_empty() {
+                        let mut create_info = ash::vk::ImageDrmFormatModifierListCreateInfoEXT {
+                            drm_format_modifier_count: modifiers_filtered.len() as u32,
+                            p_drm_format_modifiers: modifiers_filtered.as_ptr() as _,
+                            ..Default::default()
+                        };
+
+                        let vk_ptr = hwframe_casted.hwctx as *mut AVVulkanFramesContext;
+
+                        (*vk_ptr).tiling = vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT;
+                        (*vk_ptr).create_pnext = &mut create_info as *mut _ as _;
+
+                        sts = av_hwframe_ctx_init(hwframe);
+                        modifiers_filtered.pop();
+                    }
+                }
+                #[cfg(not(feature = "experimental-vulkan"))]
+                panic!("vulkan requested but built without vulkan support")
+            } else {
+                if modifiers != &[DrmModifier::LINEAR] {
+                    error!("unknown how to request non-linear frames in vaapi");
+                }
+                sts = av_hwframe_ctx_init(hwframe);
             }
-
-            let sts = av_hwframe_ctx_init(hwframe);
             if sts != 0 {
                 return Err(ffmpeg::Error::from(sts));
             }