obs-studio/patches/7206.patch

From 7ac1d462af1be795569e0d5c8f79871901e5d6e0 Mon Sep 17 00:00:00 2001
From: David Rosca <nowrep@gmail.com>
Date: Sun, 28 Aug 2022 10:15:16 +0200
Subject: [PATCH] obs-ffmpeg: Make AMF encoder work on Linux

Only the fallback encoders are available (no texture support).

Requires AMD proprietary Vulkan driver, using different driver
will be detected on startup and the encoders disabled.
---
 plugins/obs-ffmpeg/CMakeLists.txt             |   3 +
 .../obs-ffmpeg/obs-amf-test/CMakeLists.txt    |  11 +-
 .../obs-amf-test/obs-amf-test-linux.cpp       | 140 ++++++++++++++++++
 plugins/obs-ffmpeg/obs-ffmpeg.c               |  10 +-
 plugins/obs-ffmpeg/texture-amf-opts.hpp       |   2 +-
 plugins/obs-ffmpeg/texture-amf.cpp            |  81 +++++++++-
 6 files changed, 236 insertions(+), 11 deletions(-)
 create mode 100644 plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp

diff --git a/plugins/obs-ffmpeg/CMakeLists.txt b/plugins/obs-ffmpeg/CMakeLists.txt
index 681a057..35cb1ea 100644
--- a/plugins/obs-ffmpeg/CMakeLists.txt
+++ b/plugins/obs-ffmpeg/CMakeLists.txt
@@ -119,11 +119,14 @@
             obs-ffmpeg.rc)

 elseif(OS_POSIX AND NOT OS_MACOS)
+  add_subdirectory(obs-amf-test)
   find_package(Libva REQUIRED)
   find_package(Libpci REQUIRED)
   target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c
                                     vaapi-utils.h)
   target_link_libraries(obs-ffmpeg PRIVATE Libva::va Libva::drm LIBPCI::LIBPCI)
+  target_sources(obs-ffmpeg PRIVATE texture-amf.cpp)
+  target_link_libraries(obs-ffmpeg PRIVATE LIBPCI::LIBPCI)
 endif()

 setup_plugin_target(obs-ffmpeg)
diff --git a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
index 85347e7..034b4cc 100644
--- a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
+++ b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
@@ -3,8 +3,15 @@ project(obs-amf-test)
 include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/libobs)
 
 add_executable(obs-amf-test)
-target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
-target_link_libraries(obs-amf-test d3d11 dxgi dxguid)
+
+if(OS_WINDOWS)
+  target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
+  target_link_libraries(obs-amf-test d3d11 dxgi dxguid)
+elseif(OS_POSIX AND NOT OS_MACOS)
+  find_package(Vulkan REQUIRED)
+  target_sources(obs-amf-test PRIVATE obs-amf-test-linux.cpp)
+  target_link_libraries(obs-amf-test dl Vulkan::Vulkan)
+endif()
 
 set_target_properties(obs-amf-test PROPERTIES FOLDER "plugins/obs-ffmpeg")
 
diff --git a/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp b/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp
new file mode 100644
index 0000000..018486e
--- /dev/null
+++ b/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp
@@ -0,0 +1,140 @@
+#include "../external/AMF/include/core/Factory.h"
+#include "../external/AMF/include/core/Trace.h"
+#include "../external/AMF/include/components/VideoEncoderVCE.h"
+#include "../external/AMF/include/components/VideoEncoderHEVC.h"
+#include "../external/AMF/include/components/VideoEncoderAV1.h"
+
+#include <dlfcn.h>
+#include <vulkan/vulkan.hpp>
+
+#include <string>
+#include <map>
+
+using namespace amf;
+
+struct adapter_caps {
+	bool is_amd = false;
+	bool supports_avc = false;
+	bool supports_hevc = false;
+	bool supports_av1 = false;
+};
+
+static AMFFactory *amf_factory = nullptr;
+static std::map<uint32_t, adapter_caps> adapter_info;
+
+static bool has_encoder(AMFContextPtr &amf_context, const wchar_t *encoder_name)
+{
+	AMFComponentPtr encoder;
+	AMF_RESULT res = amf_factory->CreateComponent(amf_context, encoder_name,
+						      &encoder);
+	return res == AMF_OK;
+}
+
+static bool get_adapter_caps(uint32_t adapter_idx)
+{
+	if (adapter_idx)
+		return false;
+
+	adapter_caps &caps = adapter_info[adapter_idx];
+
+	AMF_RESULT res;
+	AMFContextPtr amf_context;
+	res = amf_factory->CreateContext(&amf_context);
+	if (res != AMF_OK)
+		return true;
+
+	AMFContext1 *context1 = NULL;
+	res = amf_context->QueryInterface(AMFContext1::IID(),
+					  (void **)&context1);
+	if (res != AMF_OK)
+		return false;
+	res = context1->InitVulkan(nullptr);
+	context1->Release();
+	if (res != AMF_OK)
+		return false;
+
+	caps.is_amd = true;
+	caps.supports_avc = has_encoder(amf_context, AMFVideoEncoderVCE_AVC);
+	caps.supports_hevc = has_encoder(amf_context, AMFVideoEncoder_HEVC);
+	caps.supports_av1 = has_encoder(amf_context, AMFVideoEncoder_AV1);
+
+	return true;
+}
+
+int main(void)
+try {
+	AMF_RESULT res;
+	VkResult vkres;
+
+	VkApplicationInfo app_info = {};
+	app_info.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
+	app_info.pApplicationName = "obs-amf-test";
+	app_info.apiVersion = VK_API_VERSION_1_2;
+
+	VkInstanceCreateInfo info = {};
+	info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
+	info.pApplicationInfo = &app_info;
+
+	VkInstance instance;
+	vkres = vkCreateInstance(&info, nullptr, &instance);
+	if (vkres != VK_SUCCESS)
+		throw "Failed to initialize Vulkan";
+
+	uint32_t device_count;
+	vkres = vkEnumeratePhysicalDevices(instance, &device_count, nullptr);
+	if (vkres != VK_SUCCESS || !device_count)
+		throw "Failed to enumerate Vulkan devices";
+
+	VkPhysicalDevice *devices = new VkPhysicalDevice[device_count];
+	vkres = vkEnumeratePhysicalDevices(instance, &device_count, devices);
+	if (vkres != VK_SUCCESS)
+		throw "Failed to enumerate Vulkan devices";
+
+	VkPhysicalDeviceDriverProperties driver_props = {};
+	driver_props.sType =
+		VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
+	VkPhysicalDeviceProperties2 device_props = {};
+	device_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
+	device_props.pNext = &driver_props;
+	vkGetPhysicalDeviceProperties2(devices[0], &device_props);
+
+	if (strcmp(driver_props.driverName, "AMD proprietary driver"))
+		throw "Not running AMD proprietary driver";
+
+	vkDestroyInstance(instance, nullptr);
+
+	/* --------------------------------------------------------- */
+	/* try initializing amf, I guess                             */
+
+	void *amf_module = dlopen(AMF_DLL_NAMEA, RTLD_LAZY);
+	if (!amf_module)
+		throw "Failed to load AMF lib";
+
+	auto init = (AMFInit_Fn)dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
+	if (!init)
+		throw "Failed to get init func";
+
+	res = init(AMF_FULL_VERSION, &amf_factory);
+	if (res != AMF_OK)
+		throw "AMFInit failed";
+
+	uint32_t idx = 0;
+	while (get_adapter_caps(idx++))
+		;
+
+	for (auto &[idx, caps] : adapter_info) {
+		printf("[%u]\n", idx);
+		printf("is_amd=%s\n", caps.is_amd ? "true" : "false");
+		printf("supports_avc=%s\n",
+		       caps.supports_avc ? "true" : "false");
+		printf("supports_hevc=%s\n",
+		       caps.supports_hevc ? "true" : "false");
+		printf("supports_av1=%s\n",
+		       caps.supports_av1 ? "true" : "false");
+	}
+
+	return 0;
+} catch (const char *text) {
+	printf("[error]\nstring=%s\n", text);
+	return 0;
+}
diff --git a/plugins/obs-ffmpeg/obs-ffmpeg.c b/plugins/obs-ffmpeg/obs-ffmpeg.c
index 1677e08..4bb870a 100644
--- a/plugins/obs-ffmpeg/obs-ffmpeg.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg.c
@@ -322,6 +322,9 @@ static bool nvenc_supported(bool *out_h264, bool *out_hevc, bool *out_av1)
 #ifdef _WIN32
 extern void jim_nvenc_load(bool h264, bool hevc, bool av1);
 extern void jim_nvenc_unload(void);
+#endif
+
+#if defined(_WIN32) || defined(__linux__)
 extern void amf_load(void);
 extern void amf_unload(void);
 #endif
@@ -391,7 +394,7 @@ bool obs_module_load(void)
 #endif
 	}
 
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
 	amf_load();
 #endif
 
@@ -409,8 +412,11 @@ void obs_module_unload(void)
 	obs_ffmpeg_unload_logging();
 #endif
 
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
 	amf_unload();
+#endif
+
+#ifdef _WIN32
 	jim_nvenc_unload();
 #endif
 }
diff --git a/plugins/obs-ffmpeg/texture-amf-opts.hpp b/plugins/obs-ffmpeg/texture-amf-opts.hpp
index b1c37d2..d28e3f7 100644
--- a/plugins/obs-ffmpeg/texture-amf-opts.hpp
+++ b/plugins/obs-ffmpeg/texture-amf-opts.hpp
@@ -321,7 +321,7 @@ static void amf_apply_opt(amf_base *enc, obs_option *opt)
 			val = atoi(opt->value);
 		}
 
-		os_utf8_to_wcs(opt->name, 0, wname, _countof(wname));
+		os_utf8_to_wcs(opt->name, 0, wname, amf_countof(wname));
 		if (is_bool) {
 			bool bool_val = (bool)val;
 			set_amf_property(enc, wname, bool_val);
diff --git a/plugins/obs-ffmpeg/texture-amf.cpp b/plugins/obs-ffmpeg/texture-amf.cpp
index 92e57e0..8c648ce 100644
--- a/plugins/obs-ffmpeg/texture-amf.cpp
+++ b/plugins/obs-ffmpeg/texture-amf.cpp
@@ -20,6 +20,7 @@
 #include "external/AMF/include/core/Factory.h"
 #include "external/AMF/include/core/Trace.h"
 
+#ifdef _WIN32
 #include <dxgi.h>
 #include <d3d11.h>
 #include <d3d11_1.h>
@@ -27,6 +28,8 @@
 #include <util/windows/device-enum.h>
 #include <util/windows/HRError.hpp>
 #include <util/windows/ComPtr.hpp>
+#endif
+
 #include <util/platform.h>
 #include <util/util.hpp>
 #include <util/pipe.h>
@@ -57,8 +60,10 @@ struct amf_error {
 
 struct handle_tex {
 	uint32_t handle;
+#ifdef _WIN32
 	ComPtr<ID3D11Texture2D> tex;
 	ComPtr<IDXGIKeyedMutex> km;
+#endif
 };
 
 struct adapter_caps {
@@ -74,7 +79,7 @@ static std::map<uint32_t, adapter_caps> caps;
 static bool h264_supported = false;
 static AMFFactory *amf_factory = nullptr;
 static AMFTrace *amf_trace = nullptr;
-static HMODULE amf_module = nullptr;
+static void *amf_module = nullptr;
 static uint64_t amf_version = 0;
 
 /* ========================================================================= */
@@ -122,9 +127,11 @@ struct amf_base {
 	virtual void init() = 0;
 };
 
-using d3dtex_t = ComPtr<ID3D11Texture2D>;
 using buf_t = std::vector<uint8_t>;
 
+#ifdef _WIN32
+using d3dtex_t = ComPtr<ID3D11Texture2D>;
+
 struct amf_texencode : amf_base, public AMFSurfaceObserver {
 	volatile bool destroying = false;
 
@@ -161,6 +168,7 @@ struct amf_texencode : amf_base, public AMFSurfaceObserver {
 			throw amf_error("InitDX11 failed", res);
 	}
 };
+#endif
 
 struct amf_fallback : amf_base, public AMFSurfaceObserver {
 	volatile bool destroying = false;
@@ -188,9 +196,21 @@ struct amf_fallback : amf_base, public AMFSurfaceObserver {
 
 	void init() override
 	{
+#if defined(_WIN32)
 		AMF_RESULT res = amf_context->InitDX11(nullptr, AMF_DX11_1);
 		if (res != AMF_OK)
 			throw amf_error("InitDX11 failed", res);
+#elif defined(__linux__)
+		AMFContext1 *context1 = NULL;
+		AMF_RESULT res = amf_context->QueryInterface(
+			AMFContext1::IID(), (void **)&context1);
+		if (res != AMF_OK)
+			throw amf_error("CreateContext1 failed", res);
+		res = context1->InitVulkan(nullptr);
+		context1->Release();
+		if (res != AMF_OK)
+			throw amf_error("InitVulkan failed", res);
+#endif
 	}
 };
 
@@ -249,6 +269,7 @@ static void set_amf_property(amf_base *enc, const wchar_t *name, const T &value)
 /* ------------------------------------------------------------------------- */
 /* Implementation                                                            */
 
+#ifdef _WIN32
 static HMODULE get_lib(const char *lib)
 {
 	HMODULE mod = GetModuleHandleA(lib);
@@ -395,6 +416,7 @@ static void get_tex_from_handle(amf_texencode *enc, uint32_t handle,
 	*km_out = km.Detach();
 	*tex_out = tex.Detach();
 }
+#endif
 
 static constexpr amf_int64 macroblock_size = 16;
 
@@ -640,6 +662,7 @@ static void amf_encode_base(amf_base *enc, AMFSurface *amf_surf,
 static bool amf_encode_tex(void *data, uint32_t handle, int64_t pts,
 			   uint64_t lock_key, uint64_t *next_key,
 			   encoder_packet *packet, bool *received_packet)
+#ifdef _WIN32
 try {
 	amf_texencode *enc = (amf_texencode *)data;
 	ID3D11DeviceContext *context = enc->context;
@@ -716,6 +739,18 @@ try {
 	*received_packet = false;
 	return false;
 }
+#else
+{
+	UNUSED_PARAMETER(data);
+	UNUSED_PARAMETER(handle);
+	UNUSED_PARAMETER(pts);
+	UNUSED_PARAMETER(lock_key);
+	UNUSED_PARAMETER(next_key);
+	UNUSED_PARAMETER(packet);
+	UNUSED_PARAMETER(received_packet);
+	return false;
+}
+#endif
 
 static buf_t alloc_buf(amf_fallback *enc)
 {
@@ -1406,6 +1441,7 @@ static void amf_avc_create_internal(amf_base *enc, obs_data_t *settings)
 
 static void *amf_avc_create_texencode(obs_data_t *settings,
 				      obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::AVC);
 
@@ -1428,6 +1464,12 @@ try {
 	blog(LOG_ERROR, "[texture-amf-h264] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
+}
+#endif
 
 static void *amf_avc_create_fallback(obs_data_t *settings,
 				     obs_encoder_t *encoder)
@@ -1748,6 +1790,7 @@ static void amf_hevc_create_internal(amf_base *enc, obs_data_t *settings)
 
 static void *amf_hevc_create_texencode(obs_data_t *settings,
 				       obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::HEVC);
 
@@ -1770,6 +1813,12 @@ try {
 	blog(LOG_ERROR, "[texture-amf-h265] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
+}
+#endif
 
 static void *amf_hevc_create_fallback(obs_data_t *settings,
 				      obs_encoder_t *encoder)
@@ -2047,6 +2096,7 @@ static void amf_av1_create_internal(amf_base *enc, obs_data_t *settings)
 
 static void *amf_av1_create_texencode(obs_data_t *settings,
 				      obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::AV1);
 
@@ -2069,6 +2119,12 @@ try {
 	blog(LOG_ERROR, "[texture-amf-av1] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
+}
+#endif
 
 static void *amf_av1_create_fallback(obs_data_t *settings,
 				     obs_encoder_t *encoder)
@@ -2159,9 +2215,16 @@ static bool enum_luids(void *param, uint32_t idx, uint64_t luid)
 	return true;
 }
 
+#ifdef _WIN32
+#define OBS_AMF_TEST "obs-amf-test.exe"
+#else
+#define OBS_AMF_TEST "obs-amf-test"
+#endif
+
 extern "C" void amf_load(void)
 try {
 	AMF_RESULT res;
+#ifdef _WIN32
 	HMODULE amf_module_test;
 
 	/* Check if the DLL is present before running the more expensive */
@@ -2171,17 +2234,25 @@ try {
 	if (!amf_module_test)
 		throw "No AMF library";
 	FreeLibrary(amf_module_test);
+#else
+	void *amf_module_test = os_dlopen(AMF_DLL_NAMEA);
+	if (!amf_module_test)
+		throw "No AMF library";
+	os_dlclose(amf_module_test);
+#endif
 
 	/* ----------------------------------- */
 	/* Check for supported codecs          */
 
-	BPtr<char> test_exe = os_get_executable_path_ptr("obs-amf-test.exe");
+	BPtr<char> test_exe = os_get_executable_path_ptr(OBS_AMF_TEST);
 	std::stringstream cmd;
 	std::string caps_str;
 
 	cmd << test_exe;
-	enum_graphics_device_luids(enum_luids, &cmd);
 
+#ifdef _WIN32
+	enum_graphics_device_luids(enum_luids, &cmd);
+#endif
 	os_process_pipe_t *pp = os_process_pipe_create(cmd.str().c_str(), "r");
 	if (!pp)
 		throw "Failed to launch the AMF test process I guess";
@@ -2240,12 +2311,12 @@ try {
 	/* ----------------------------------- */
 	/* Init AMF                            */
 
-	amf_module = LoadLibraryW(AMF_DLL_NAME);
+	amf_module = os_dlopen(AMF_DLL_NAMEA);
 	if (!amf_module)
 		throw "AMF library failed to load";
 
 	AMFInit_Fn init =
-		(AMFInit_Fn)GetProcAddress(amf_module, AMF_INIT_FUNCTION_NAME);
+		(AMFInit_Fn)os_dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
 	if (!init)
 		throw "Failed to get AMFInit address";
 
@@ -2257,7 +2328,7 @@ try {
 	if (res != AMF_OK)
 		throw amf_error("GetTrace failed", res);
 
-	AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)GetProcAddress(
+	AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)os_dlsym(
 		amf_module, AMF_QUERY_VERSION_FUNCTION_NAME);
 	if (!get_ver)
 		throw "Failed to get AMFQueryVersion address";
-- 
2.39.0