obs-studio/patches/04_AMF-encoder.patch
Ward from fusion-voyager-3 951709f276
Some checks failed
PikaOS Package Build Only (Canary) (amd64-v3) / build (push) Has been cancelled
PikaOS Package Build Only (amd64-v3) / build (push) Failing after 0s
PikaOS Package Build & Release (Canary) (amd64-v3) / build (push) Failing after 39s
PikaOS Package Build & Release (amd64-v3) / build (push) Failing after 0s
port to pika os 4
2024-07-30 20:49:10 +03:00

613 lines
19 KiB
Diff

From dbd2bca8060de57c3a8309de38556371aa60182f Mon Sep 17 00:00:00 2001
From: David Rosca <nowrep@gmail.com>
Date: Sun, 28 Aug 2022 10:15:16 +0200
Subject: [PATCH] obs-ffmpeg: Make AMF encoder work on Linux
Only the fallback encoders are available (no texture support).
Requires AMD proprietary Vulkan driver, using different driver
will be detected on startup and the encoders disabled.
---
--- a/plugins/obs-ffmpeg/CMakeLists.txt
+++ b/plugins/obs-ffmpeg/CMakeLists.txt
@@ -72,7 +72,10 @@ target_link_libraries(
if(OS_WINDOWS)
configure_file(cmake/windows/obs-module.rc.in obs-ffmpeg.rc)
- target_sources(obs-ffmpeg PRIVATE obs-ffmpeg.rc)
+ target_sources(obs-ffmpeg PRIVATE obs-ffmpeg.rc texture-amf.cpp)
+
+elseif(OS_LINUX OR OS_FREEBSD)
+ add_subdirectory(obs-amf-test)
endif()
# cmake-format: off
--- a/plugins/obs-ffmpeg/cmake/legacy.cmake
+++ b/plugins/obs-ffmpeg/cmake/legacy.cmake
@@ -109,10 +109,11 @@ if(OS_WINDOWS)
obs-ffmpeg.rc)
elseif(OS_POSIX AND NOT OS_MACOS)
+ add_subdirectory(obs-amf-test)
find_package(Libva REQUIRED)
find_package(Libpci REQUIRED)
find_package(Libdrm REQUIRED)
- target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c vaapi-utils.h)
+ target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c vaapi-utils.h texture-amf.cpp)
target_link_libraries(obs-ffmpeg PRIVATE Libva::va Libva::drm LIBPCI::LIBPCI Libdrm::Libdrm)
if(ENABLE_NATIVE_NVENC)
--- a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
+++ b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
@@ -1,15 +1,20 @@
-cmake_minimum_required(VERSION 3.24...3.25)
+project(obs-amf-test)
-legacy_check()
+add_executable(obs-amf-test)
find_package(AMF 1.4.29 REQUIRED)
-add_executable(obs-amf-test)
-add_executable(OBS::amf-test ALIAS obs-amf-test)
+target_include_directories(obs-amf-test PRIVATE ${CMAKE_SOURCE_DIR}/libobs)
+
+if(OS_WINDOWS)
+ target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
+ target_link_libraries(obs-amf-test d3d11 dxgi dxguid AMF::AMF)
+elseif(OS_POSIX AND NOT OS_MACOS)
+ find_package(Vulkan REQUIRED)
+ target_sources(obs-amf-test PRIVATE obs-amf-test-linux.cpp)
+ target_link_libraries(obs-amf-test dl Vulkan::Vulkan AMF::AMF)
+endif()
-target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
-target_link_libraries(obs-amf-test PRIVATE OBS::COMutils AMF::AMF d3d11 dxgi dxguid)
+set_target_properties(obs-amf-test PROPERTIES FOLDER "plugins/obs-ffmpeg")
-# cmake-format: off
-set_target_properties_obs(obs-amf-test PROPERTIES FOLDER plugins/obs-ffmpeg)
-# cmake-format: on
+setup_binary_target(obs-amf-test)
--- /dev/null
+++ b/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp
@@ -0,0 +1,140 @@
+#include <AMF/core/Factory.h>
+#include <AMF/core/Trace.h>
+#include <AMF/components/VideoEncoderVCE.h>
+#include <AMF/components/VideoEncoderHEVC.h>
+#include <AMF/components/VideoEncoderAV1.h>
+
+#include <dlfcn.h>
+#include <vulkan/vulkan.hpp>
+
+#include <string>
+#include <map>
+
+using namespace amf;
+
+struct adapter_caps {
+ bool is_amd = false;
+ bool supports_avc = false;
+ bool supports_hevc = false;
+ bool supports_av1 = false;
+};
+
+static AMFFactory *amf_factory = nullptr;
+static std::map<uint32_t, adapter_caps> adapter_info;
+
+static bool has_encoder(AMFContextPtr &amf_context, const wchar_t *encoder_name)
+{
+ AMFComponentPtr encoder;
+ AMF_RESULT res = amf_factory->CreateComponent(amf_context, encoder_name,
+ &encoder);
+ return res == AMF_OK;
+}
+
+static bool get_adapter_caps(uint32_t adapter_idx)
+{
+ if (adapter_idx)
+ return false;
+
+ adapter_caps &caps = adapter_info[adapter_idx];
+
+ AMF_RESULT res;
+ AMFContextPtr amf_context;
+ res = amf_factory->CreateContext(&amf_context);
+ if (res != AMF_OK)
+ return true;
+
+ AMFContext1 *context1 = NULL;
+ res = amf_context->QueryInterface(AMFContext1::IID(),
+ (void **)&context1);
+ if (res != AMF_OK)
+ return false;
+ res = context1->InitVulkan(nullptr);
+ context1->Release();
+ if (res != AMF_OK)
+ return false;
+
+ caps.is_amd = true;
+ caps.supports_avc = has_encoder(amf_context, AMFVideoEncoderVCE_AVC);
+ caps.supports_hevc = has_encoder(amf_context, AMFVideoEncoder_HEVC);
+ caps.supports_av1 = has_encoder(amf_context, AMFVideoEncoder_AV1);
+
+ return true;
+}
+
+int main(void)
+try {
+ AMF_RESULT res;
+ VkResult vkres;
+
+ VkApplicationInfo app_info = {};
+ app_info.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
+ app_info.pApplicationName = "obs-amf-test";
+ app_info.apiVersion = VK_API_VERSION_1_2;
+
+ VkInstanceCreateInfo info = {};
+ info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
+ info.pApplicationInfo = &app_info;
+
+ VkInstance instance;
+ vkres = vkCreateInstance(&info, nullptr, &instance);
+ if (vkres != VK_SUCCESS)
+ throw "Failed to initialize Vulkan";
+
+ uint32_t device_count;
+ vkres = vkEnumeratePhysicalDevices(instance, &device_count, nullptr);
+ if (vkres != VK_SUCCESS || !device_count)
+ throw "Failed to enumerate Vulkan devices";
+
+ VkPhysicalDevice *devices = new VkPhysicalDevice[device_count];
+ vkres = vkEnumeratePhysicalDevices(instance, &device_count, devices);
+ if (vkres != VK_SUCCESS)
+ throw "Failed to enumerate Vulkan devices";
+
+ VkPhysicalDeviceDriverProperties driver_props = {};
+ driver_props.sType =
+ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
+ VkPhysicalDeviceProperties2 device_props = {};
+ device_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
+ device_props.pNext = &driver_props;
+ vkGetPhysicalDeviceProperties2(devices[0], &device_props);
+
+ if (strcmp(driver_props.driverName, "AMD proprietary driver"))
+ throw "Not running AMD proprietary driver";
+
+ vkDestroyInstance(instance, nullptr);
+
+ /* --------------------------------------------------------- */
+ /* try initializing amf, I guess */
+
+ void *amf_module = dlopen(AMF_DLL_NAMEA, RTLD_LAZY);
+ if (!amf_module)
+ throw "Failed to load AMF lib";
+
+ auto init = (AMFInit_Fn)dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
+ if (!init)
+ throw "Failed to get init func";
+
+ res = init(AMF_FULL_VERSION, &amf_factory);
+ if (res != AMF_OK)
+ throw "AMFInit failed";
+
+ uint32_t idx = 0;
+ while (get_adapter_caps(idx++))
+ ;
+
+ for (auto &[idx, caps] : adapter_info) {
+ printf("[%u]\n", idx);
+ printf("is_amd=%s\n", caps.is_amd ? "true" : "false");
+ printf("supports_avc=%s\n",
+ caps.supports_avc ? "true" : "false");
+ printf("supports_hevc=%s\n",
+ caps.supports_hevc ? "true" : "false");
+ printf("supports_av1=%s\n",
+ caps.supports_av1 ? "true" : "false");
+ }
+
+ return 0;
+} catch (const char *text) {
+ printf("[error]\nstring=%s\n", text);
+ return 0;
+}
--- a/plugins/obs-ffmpeg/obs-ffmpeg.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg.c
@@ -393,7 +393,7 @@ bool obs_module_load(void)
#endif
}
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
amf_load();
#endif
--- a/plugins/obs-ffmpeg/texture-amf-opts.hpp
+++ b/plugins/obs-ffmpeg/texture-amf-opts.hpp
@@ -321,7 +321,7 @@ static void amf_apply_opt(amf_base *enc,
val = atoi(opt->value);
}
- os_utf8_to_wcs(opt->name, 0, wname, _countof(wname));
+ os_utf8_to_wcs(opt->name, 0, wname, amf_countof(wname));
if (is_bool) {
bool bool_val = (bool)val;
set_amf_property(enc, wname, bool_val);
--- a/plugins/obs-ffmpeg/texture-amf.cpp
+++ b/plugins/obs-ffmpeg/texture-amf.cpp
@@ -11,6 +11,7 @@
#include <mutex>
#include <deque>
#include <map>
+#include <inttypes.h>
#include <AMF/components/VideoEncoderHEVC.h>
#include <AMF/components/VideoEncoderVCE.h>
@@ -18,6 +19,7 @@
#include <AMF/core/Factory.h>
#include <AMF/core/Trace.h>
+#ifdef _WIN32
#include <dxgi.h>
#include <d3d11.h>
#include <d3d11_1.h>
@@ -25,6 +27,8 @@
#include <util/windows/device-enum.h>
#include <util/windows/HRError.hpp>
#include <util/windows/ComPtr.hpp>
+#endif
+
#include <util/platform.h>
#include <util/util.hpp>
#include <util/pipe.h>
@@ -55,8 +59,10 @@ struct amf_error {
struct handle_tex {
uint32_t handle;
+#ifdef _WIN32
ComPtr<ID3D11Texture2D> tex;
ComPtr<IDXGIKeyedMutex> km;
+#endif
};
struct adapter_caps {
@@ -72,7 +78,7 @@ static std::map<uint32_t, adapter_caps>
static bool h264_supported = false;
static AMFFactory *amf_factory = nullptr;
static AMFTrace *amf_trace = nullptr;
-static HMODULE amf_module = nullptr;
+static void *amf_module = nullptr;
static uint64_t amf_version = 0;
/* ========================================================================= */
@@ -124,9 +130,11 @@ struct amf_base {
virtual void init() = 0;
};
-using d3dtex_t = ComPtr<ID3D11Texture2D>;
using buf_t = std::vector<uint8_t>;
+#ifdef _WIN32
+using d3dtex_t = ComPtr<ID3D11Texture2D>;
+
struct amf_texencode : amf_base, public AMFSurfaceObserver {
volatile bool destroying = false;
@@ -163,6 +171,7 @@ struct amf_texencode : amf_base, public
throw amf_error("InitDX11 failed", res);
}
};
+#endif
struct amf_fallback : amf_base, public AMFSurfaceObserver {
volatile bool destroying = false;
@@ -190,9 +199,21 @@ struct amf_fallback : amf_base, public A
void init() override
{
+#if defined(_WIN32)
AMF_RESULT res = amf_context->InitDX11(nullptr, AMF_DX11_1);
if (res != AMF_OK)
throw amf_error("InitDX11 failed", res);
+#elif defined(__linux__)
+ AMFContext1 *context1 = NULL;
+ AMF_RESULT res = amf_context->QueryInterface(
+ AMFContext1::IID(), (void **)&context1);
+ if (res != AMF_OK)
+ throw amf_error("CreateContext1 failed", res);
+ res = context1->InitVulkan(nullptr);
+ context1->Release();
+ if (res != AMF_OK)
+ throw amf_error("InitVulkan failed", res);
+#endif
}
};
@@ -234,13 +255,18 @@ static void set_amf_property(amf_base *e
: (enc->codec == amf_codec_type::HEVC) \
? AMF_VIDEO_ENCODER_HEVC_##name \
: AMF_VIDEO_ENCODER_AV1_##name)
+#define get_opt_name_enum(name) \
+ ((enc->codec == amf_codec_type::AVC) ? (int)AMF_VIDEO_ENCODER_##name \
+ : (enc->codec == amf_codec_type::HEVC) \
+ ? (int)AMF_VIDEO_ENCODER_HEVC_##name \
+ : (int)AMF_VIDEO_ENCODER_AV1_##name)
#define set_opt(name, value) set_amf_property(enc, get_opt_name(name), value)
#define get_opt(name, value) get_amf_property(enc, get_opt_name(name), value)
#define set_avc_opt(name, value) set_avc_property(enc, name, value)
#define set_hevc_opt(name, value) set_hevc_property(enc, name, value)
#define set_av1_opt(name, value) set_av1_property(enc, name, value)
#define set_enum_opt(name, value) \
- set_amf_property(enc, get_opt_name(name), get_opt_name(name##_##value))
+ set_amf_property(enc, get_opt_name(name), get_opt_name_enum(name##_##value))
#define set_avc_enum(name, value) \
set_avc_property(enc, name, AMF_VIDEO_ENCODER_##name##_##value)
#define set_hevc_enum(name, value) \
@@ -251,6 +277,7 @@ static void set_amf_property(amf_base *e
/* ------------------------------------------------------------------------- */
/* Implementation */
+#ifdef _WIN32
static HMODULE get_lib(const char *lib)
{
HMODULE mod = GetModuleHandleA(lib);
@@ -397,6 +424,7 @@ static void get_tex_from_handle(amf_texe
*km_out = km.Detach();
*tex_out = tex.Detach();
}
+#endif
static constexpr amf_int64 macroblock_size = 16;
@@ -513,7 +541,7 @@ static void convert_to_encoder_packet(am
enc->packet_data = AMFBufferPtr(data);
data->GetProperty(L"PTS", &packet->pts);
- const wchar_t *get_output_type;
+ const wchar_t *get_output_type = NULL;
switch (enc->codec) {
case amf_codec_type::AVC:
get_output_type = AMF_VIDEO_ENCODER_OUTPUT_DATA_TYPE;
@@ -746,6 +774,7 @@ static void amf_encode_base(amf_base *en
static bool amf_encode_tex(void *data, uint32_t handle, int64_t pts,
uint64_t lock_key, uint64_t *next_key,
encoder_packet *packet, bool *received_packet)
+#ifdef _WIN32
try {
amf_texencode *enc = (amf_texencode *)data;
ID3D11DeviceContext *context = enc->context;
@@ -822,6 +851,18 @@ try {
*received_packet = false;
return false;
}
+#else
+{
+ UNUSED_PARAMETER(data);
+ UNUSED_PARAMETER(handle);
+ UNUSED_PARAMETER(pts);
+ UNUSED_PARAMETER(lock_key);
+ UNUSED_PARAMETER(next_key);
+ UNUSED_PARAMETER(packet);
+ UNUSED_PARAMETER(received_packet);
+ return false;
+}
+#endif
static buf_t alloc_buf(amf_fallback *enc)
{
@@ -1286,6 +1327,7 @@ static const char *amf_avc_get_name(void
static inline int get_avc_preset(amf_base *enc, const char *preset)
{
+ UNUSED_PARAMETER(enc);
if (astrcmpi(preset, "quality") == 0)
return AMF_VIDEO_ENCODER_QUALITY_PRESET_QUALITY;
else if (astrcmpi(preset, "speed") == 0)
@@ -1408,7 +1450,7 @@ static bool amf_avc_init(void *data, obs
}
} else if (bf != 0) {
- warn("B-Frames set to %lld but b-frames are not "
+ warn("B-Frames set to %" PRId64 " but b-frames are not "
"supported by this device",
bf);
bf = 0;
@@ -1453,12 +1495,12 @@ static bool amf_avc_init(void *data, obs
info("settings:\n"
"\trate_control: %s\n"
- "\tbitrate: %d\n"
- "\tcqp: %d\n"
+ "\tbitrate: %" PRId64 "\n"
+ "\tcqp: %" PRId64 "\n"
"\tkeyint: %d\n"
"\tpreset: %s\n"
"\tprofile: %s\n"
- "\tb-frames: %d\n"
+ "\tb-frames: %" PRId64 "\n"
"\twidth: %d\n"
"\theight: %d\n"
"\tparams: %s",
@@ -1531,6 +1573,7 @@ static void amf_avc_create_internal(amf_
static void *amf_avc_create_texencode(obs_data_t *settings,
obs_encoder_t *encoder)
+#ifdef _WIN32
try {
check_texture_encode_capability(encoder, amf_codec_type::AVC);
@@ -1553,6 +1596,12 @@ try {
blog(LOG_ERROR, "[texture-amf-h264] %s: %s", __FUNCTION__, err);
return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
}
+#else
+{
+ UNUSED_PARAMETER(settings);
+ return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
+}
+#endif
static void *amf_avc_create_fallback(obs_data_t *settings,
obs_encoder_t *encoder)
@@ -1647,6 +1696,7 @@ static const char *amf_hevc_get_name(voi
static inline int get_hevc_preset(amf_base *enc, const char *preset)
{
+ UNUSED_PARAMETER(enc);
if (astrcmpi(preset, "balanced") == 0)
return AMF_VIDEO_ENCODER_HEVC_QUALITY_PRESET_BALANCED;
else if (astrcmpi(preset, "speed") == 0)
@@ -1770,8 +1820,8 @@ static bool amf_hevc_init(void *data, ob
info("settings:\n"
"\trate_control: %s\n"
- "\tbitrate: %d\n"
- "\tcqp: %d\n"
+ "\tbitrate: %" PRId64 "\n"
+ "\tcqp: %" PRId64 "\n"
"\tkeyint: %d\n"
"\tpreset: %s\n"
"\tprofile: %s\n"
@@ -1892,6 +1942,7 @@ static void amf_hevc_create_internal(amf
static void *amf_hevc_create_texencode(obs_data_t *settings,
obs_encoder_t *encoder)
+#ifdef _WIN32
try {
check_texture_encode_capability(encoder, amf_codec_type::HEVC);
@@ -1914,6 +1965,12 @@ try {
blog(LOG_ERROR, "[texture-amf-h265] %s: %s", __FUNCTION__, err);
return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
}
+#else
+{
+ UNUSED_PARAMETER(settings);
+ return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
+}
+#endif
static void *amf_hevc_create_fallback(obs_data_t *settings,
obs_encoder_t *encoder)
@@ -2004,6 +2061,7 @@ static const char *amf_av1_get_name(void
static inline int get_av1_preset(amf_base *enc, const char *preset)
{
+ UNUSED_PARAMETER(enc);
if (astrcmpi(preset, "highquality") == 0)
return AMF_VIDEO_ENCODER_AV1_QUALITY_PRESET_HIGH_QUALITY;
else if (astrcmpi(preset, "quality") == 0)
@@ -2142,8 +2200,8 @@ static bool amf_av1_init(void *data, obs
info("settings:\n"
"\trate_control: %s\n"
- "\tbitrate: %d\n"
- "\tcqp: %d\n"
+ "\tbitrate: %" PRId64 "\n"
+ "\tcqp: %" PRId64 "\n"
"\tkeyint: %d\n"
"\tpreset: %s\n"
"\tprofile: %s\n"
@@ -2211,6 +2269,7 @@ static void amf_av1_create_internal(amf_
static void *amf_av1_create_texencode(obs_data_t *settings,
obs_encoder_t *encoder)
+#ifdef _WIN32
try {
check_texture_encode_capability(encoder, amf_codec_type::AV1);
@@ -2233,6 +2292,12 @@ try {
blog(LOG_ERROR, "[texture-amf-av1] %s: %s", __FUNCTION__, err);
return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
}
+#else
+{
+ UNUSED_PARAMETER(settings);
+ return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
+}
+#endif
static void *amf_av1_create_fallback(obs_data_t *settings,
obs_encoder_t *encoder)
@@ -2332,9 +2397,16 @@ static bool enum_luids(void *param, uint
return true;
}
+#ifdef _WIN32
+#define OBS_AMF_TEST "obs-amf-test.exe"
+#else
+#define OBS_AMF_TEST "obs-amf-test"
+#endif
+
extern "C" void amf_load(void)
try {
AMF_RESULT res;
+#ifdef _WIN32
HMODULE amf_module_test;
/* Check if the DLL is present before running the more expensive */
@@ -2344,18 +2416,26 @@ try {
if (!amf_module_test)
throw "No AMF library";
FreeLibrary(amf_module_test);
+#else
+ void *amf_module_test = os_dlopen(AMF_DLL_NAMEA);
+ if (!amf_module_test)
+ throw "No AMF library";
+ os_dlclose(amf_module_test);
+#endif
/* ----------------------------------- */
/* Check for supported codecs */
- BPtr<char> test_exe = os_get_executable_path_ptr("obs-amf-test.exe");
+ BPtr<char> test_exe = os_get_executable_path_ptr(OBS_AMF_TEST);
std::stringstream cmd;
std::string caps_str;
cmd << '"';
cmd << test_exe;
cmd << '"';
+#ifdef _WIN32
enum_graphics_device_luids(enum_luids, &cmd);
+#endif
os_process_pipe_t *pp = os_process_pipe_create(cmd.str().c_str(), "r");
if (!pp)
@@ -2415,12 +2495,12 @@ try {
/* ----------------------------------- */
/* Init AMF */
- amf_module = LoadLibraryW(AMF_DLL_NAME);
+ amf_module = os_dlopen(AMF_DLL_NAMEA);
if (!amf_module)
throw "AMF library failed to load";
AMFInit_Fn init =
- (AMFInit_Fn)GetProcAddress(amf_module, AMF_INIT_FUNCTION_NAME);
+ (AMFInit_Fn)os_dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
if (!init)
throw "Failed to get AMFInit address";
@@ -2432,7 +2512,7 @@ try {
if (res != AMF_OK)
throw amf_error("GetTrace failed", res);
- AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)GetProcAddress(
+ AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)os_dlsym(
amf_module, AMF_QUERY_VERSION_FUNCTION_NAME);
if (!get_ver)
throw "Failed to get AMFQueryVersion address";
@@ -2471,7 +2551,7 @@ try {
} catch (const amf_error &err) {
/* doing an error here because it means at least the library has loaded
* successfully, so they probably have AMD at this point */
- blog(LOG_ERROR, "%s: %s: 0x%lX", __FUNCTION__, err.str,
+ blog(LOG_ERROR, "%s: %s: 0x%uX", __FUNCTION__, err.str,
(uint32_t)err.res);
}