obs-studio/patches/7206.patch

From ef97ae930812bcd418cc57403669d50d36f3e5b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Miguel=20Sarasola?= <alosarjos@gmail.com>
Date: Fri, 3 Feb 2023 23:08:46 +0100
Subject: [PATCH] Test

---
 plugins/obs-ffmpeg/CMakeLists.txt             |   3 +
 .../obs-ffmpeg/obs-amf-test/CMakeLists.txt    |  11 +-
 .../obs-amf-test/obs-amf-test-linux.cpp       | 140 ++++++++++++++++++
 plugins/obs-ffmpeg/obs-ffmpeg.c               |  10 +-
 plugins/obs-ffmpeg/texture-amf-opts.hpp       |   2 +-
 plugins/obs-ffmpeg/texture-amf.cpp            |  93 +++++++++---
 6 files changed, 236 insertions(+), 23 deletions(-)
 create mode 100644 plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp

diff --git a/plugins/obs-ffmpeg/CMakeLists.txt b/plugins/obs-ffmpeg/CMakeLists.txt
index 244c6df92..ef1cd98d4 100644
--- a/plugins/obs-ffmpeg/CMakeLists.txt
+++ b/plugins/obs-ffmpeg/CMakeLists.txt
@@ -119,11 +119,14 @@ if(OS_WINDOWS)
             obs-ffmpeg.rc)
 
 elseif(OS_POSIX AND NOT OS_MACOS)
+  add_subdirectory(obs-amf-test)
   find_package(Libva REQUIRED)
   find_package(Libpci REQUIRED)
   target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c
                                     vaapi-utils.h)
   target_link_libraries(obs-ffmpeg PRIVATE Libva::va Libva::drm LIBPCI::LIBPCI)
+  target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c texture-amf.cpp)
+  target_link_libraries(obs-ffmpeg PRIVATE LIBPCI::LIBPCI)
 endif()
 
 setup_plugin_target(obs-ffmpeg)
diff --git a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
index 85347e73d..034b4cc3b 100644
--- a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
+++ b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
@@ -3,8 +3,15 @@ project(obs-amf-test)
 include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/libobs)
 
 add_executable(obs-amf-test)
-target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
-target_link_libraries(obs-amf-test d3d11 dxgi dxguid)
+
+if(OS_WINDOWS)
+  target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
+  target_link_libraries(obs-amf-test d3d11 dxgi dxguid)
+elseif(OS_POSIX AND NOT OS_MACOS)
+  find_package(Vulkan REQUIRED)
+  target_sources(obs-amf-test PRIVATE obs-amf-test-linux.cpp)
+  target_link_libraries(obs-amf-test dl Vulkan::Vulkan)
+endif()
 
 set_target_properties(obs-amf-test PROPERTIES FOLDER "plugins/obs-ffmpeg")
 
diff --git a/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp b/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp
new file mode 100644
index 000000000..018486e0c
--- /dev/null
+++ b/plugins/obs-ffmpeg/obs-amf-test/obs-amf-test-linux.cpp
@@ -0,0 +1,140 @@
+#include "../external/AMF/include/core/Factory.h"
+#include "../external/AMF/include/core/Trace.h"
+#include "../external/AMF/include/components/VideoEncoderVCE.h"
+#include "../external/AMF/include/components/VideoEncoderHEVC.h"
+#include "../external/AMF/include/components/VideoEncoderAV1.h"
+
+#include <dlfcn.h>
+#include <vulkan/vulkan.hpp>
+
+#include <string>
+#include <map>
+
+using namespace amf;
+
+struct adapter_caps {
+	bool is_amd = false;
+	bool supports_avc = false;
+	bool supports_hevc = false;
+	bool supports_av1 = false;
+};
+
+static AMFFactory *amf_factory = nullptr;
+static std::map<uint32_t, adapter_caps> adapter_info;
+
+static bool has_encoder(AMFContextPtr &amf_context, const wchar_t *encoder_name)
+{
+	AMFComponentPtr encoder;
+	AMF_RESULT res = amf_factory->CreateComponent(amf_context, encoder_name,
+						      &encoder);
+	return res == AMF_OK;
+}
+
+static bool get_adapter_caps(uint32_t adapter_idx)
+{
+	if (adapter_idx)
+		return false;
+
+	adapter_caps &caps = adapter_info[adapter_idx];
+
+	AMF_RESULT res;
+	AMFContextPtr amf_context;
+	res = amf_factory->CreateContext(&amf_context);
+	if (res != AMF_OK)
+		return true;
+
+	AMFContext1 *context1 = NULL;
+	res = amf_context->QueryInterface(AMFContext1::IID(),
+					  (void **)&context1);
+	if (res != AMF_OK)
+		return false;
+	res = context1->InitVulkan(nullptr);
+	context1->Release();
+	if (res != AMF_OK)
+		return false;
+
+	caps.is_amd = true;
+	caps.supports_avc = has_encoder(amf_context, AMFVideoEncoderVCE_AVC);
+	caps.supports_hevc = has_encoder(amf_context, AMFVideoEncoder_HEVC);
+	caps.supports_av1 = has_encoder(amf_context, AMFVideoEncoder_AV1);
+
+	return true;
+}
+
+int main(void)
+try {
+	AMF_RESULT res;
+	VkResult vkres;
+
+	VkApplicationInfo app_info = {};
+	app_info.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
+	app_info.pApplicationName = "obs-amf-test";
+	app_info.apiVersion = VK_API_VERSION_1_2;
+
+	VkInstanceCreateInfo info = {};
+	info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
+	info.pApplicationInfo = &app_info;
+
+	VkInstance instance;
+	vkres = vkCreateInstance(&info, nullptr, &instance);
+	if (vkres != VK_SUCCESS)
+		throw "Failed to initialize Vulkan";
+
+	uint32_t device_count;
+	vkres = vkEnumeratePhysicalDevices(instance, &device_count, nullptr);
+	if (vkres != VK_SUCCESS || !device_count)
+		throw "Failed to enumerate Vulkan devices";
+
+	VkPhysicalDevice *devices = new VkPhysicalDevice[device_count];
+	vkres = vkEnumeratePhysicalDevices(instance, &device_count, devices);
+	if (vkres != VK_SUCCESS)
+		throw "Failed to enumerate Vulkan devices";
+
+	VkPhysicalDeviceDriverProperties driver_props = {};
+	driver_props.sType =
+		VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES;
+	VkPhysicalDeviceProperties2 device_props = {};
+	device_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
+	device_props.pNext = &driver_props;
+	vkGetPhysicalDeviceProperties2(devices[0], &device_props);
+
+	if (strcmp(driver_props.driverName, "AMD proprietary driver"))
+		throw "Not running AMD proprietary driver";
+
+	vkDestroyInstance(instance, nullptr);
+
+	/* --------------------------------------------------------- */
+	/* try initializing amf, I guess                             */
+
+	void *amf_module = dlopen(AMF_DLL_NAMEA, RTLD_LAZY);
+	if (!amf_module)
+		throw "Failed to load AMF lib";
+
+	auto init = (AMFInit_Fn)dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
+	if (!init)
+		throw "Failed to get init func";
+
+	res = init(AMF_FULL_VERSION, &amf_factory);
+	if (res != AMF_OK)
+		throw "AMFInit failed";
+
+	uint32_t idx = 0;
+	while (get_adapter_caps(idx++))
+		;
+
+	for (auto &[idx, caps] : adapter_info) {
+		printf("[%u]\n", idx);
+		printf("is_amd=%s\n", caps.is_amd ? "true" : "false");
+		printf("supports_avc=%s\n",
+		       caps.supports_avc ? "true" : "false");
+		printf("supports_hevc=%s\n",
+		       caps.supports_hevc ? "true" : "false");
+		printf("supports_av1=%s\n",
+		       caps.supports_av1 ? "true" : "false");
+	}
+
+	return 0;
+} catch (const char *text) {
+	printf("[error]\nstring=%s\n", text);
+	return 0;
+}
diff --git a/plugins/obs-ffmpeg/obs-ffmpeg.c b/plugins/obs-ffmpeg/obs-ffmpeg.c
index f1884c789..bce316d4f 100644
--- a/plugins/obs-ffmpeg/obs-ffmpeg.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg.c
@@ -339,6 +339,9 @@ static bool h264_vaapi_supported(void)
 #ifdef _WIN32
 extern void jim_nvenc_load(bool h264, bool hevc, bool av1);
 extern void jim_nvenc_unload(void);
+#endif
+
+#if defined(_WIN32) || defined(__linux__)
 extern void amf_load(void);
 extern void amf_unload(void);
 #endif
@@ -408,7 +411,7 @@ bool obs_module_load(void)
 #endif
 	}
 
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
 	amf_load();
 #endif
 
@@ -440,8 +443,11 @@ void obs_module_unload(void)
 	obs_ffmpeg_unload_logging();
 #endif
 
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
 	amf_unload();
+#endif
+
+#ifdef _WIN32
 	jim_nvenc_unload();
 #endif
 }
diff --git a/plugins/obs-ffmpeg/texture-amf-opts.hpp b/plugins/obs-ffmpeg/texture-amf-opts.hpp
index b1c37d200..d28e3f77e 100644
--- a/plugins/obs-ffmpeg/texture-amf-opts.hpp
+++ b/plugins/obs-ffmpeg/texture-amf-opts.hpp
@@ -321,7 +321,7 @@ static void amf_apply_opt(amf_base *enc, obs_option *opt)
 			val = atoi(opt->value);
 		}
 
-		os_utf8_to_wcs(opt->name, 0, wname, _countof(wname));
+		os_utf8_to_wcs(opt->name, 0, wname, amf_countof(wname));
 		if (is_bool) {
 			bool bool_val = (bool)val;
 			set_amf_property(enc, wname, bool_val);
diff --git a/plugins/obs-ffmpeg/texture-amf.cpp b/plugins/obs-ffmpeg/texture-amf.cpp
index 7916cf85d..201be298c 100644
--- a/plugins/obs-ffmpeg/texture-amf.cpp
+++ b/plugins/obs-ffmpeg/texture-amf.cpp
@@ -20,6 +20,7 @@
 #include "external/AMF/include/core/Factory.h"
 #include "external/AMF/include/core/Trace.h"
 
+#ifdef _WIN32
 #include <dxgi.h>
 #include <d3d11.h>
 #include <d3d11_1.h>
@@ -27,6 +28,8 @@
 #include <util/windows/device-enum.h>
 #include <util/windows/HRError.hpp>
 #include <util/windows/ComPtr.hpp>
+#endif
+
 #include <util/platform.h>
 #include <util/util.hpp>
 #include <util/pipe.h>
@@ -57,8 +60,10 @@ struct amf_error {
 
 struct handle_tex {
 	uint32_t handle;
+#ifdef _WIN32
 	ComPtr<ID3D11Texture2D> tex;
 	ComPtr<IDXGIKeyedMutex> km;
+#endif
 };
 
 struct adapter_caps {
@@ -74,7 +79,7 @@ static std::map<uint32_t, adapter_caps> caps;
 static bool h264_supported = false;
 static AMFFactory *amf_factory = nullptr;
 static AMFTrace *amf_trace = nullptr;
-static HMODULE amf_module = nullptr;
+static void *amf_module = nullptr;
 static uint64_t amf_version = 0;
 
 /* ========================================================================= */
@@ -122,9 +127,11 @@ struct amf_base {
 	virtual void init() = 0;
 };
 
-using d3dtex_t = ComPtr<ID3D11Texture2D>;
 using buf_t = std::vector<uint8_t>;
 
+#ifdef _WIN32
+using d3dtex_t = ComPtr<ID3D11Texture2D>;
+
 struct amf_texencode : amf_base, public AMFSurfaceObserver {
 	volatile bool destroying = false;
 
@@ -161,6 +168,7 @@ struct amf_texencode : amf_base, public AMFSurfaceObserver {
 			throw amf_error("InitDX11 failed", res);
 	}
 };
+#endif
 
 struct amf_fallback : amf_base, public AMFSurfaceObserver {
 	volatile bool destroying = false;
@@ -188,9 +196,21 @@ struct amf_fallback : amf_base, public AMFSurfaceObserver {
 
 	void init() override
 	{
+#if defined(_WIN32)
 		AMF_RESULT res = amf_context->InitDX11(nullptr, AMF_DX11_1);
 		if (res != AMF_OK)
 			throw amf_error("InitDX11 failed", res);
+#elif defined(__linux__)
+		AMFContext1 *context1 = NULL;
+		AMF_RESULT res = amf_context->QueryInterface(
+			AMFContext1::IID(), (void **)&context1);
+		if (res != AMF_OK)
+			throw amf_error("CreateContext1 failed", res);
+		res = context1->InitVulkan(nullptr);
+		context1->Release();
+		if (res != AMF_OK)
+			throw amf_error("InitVulkan failed", res);
+#endif
 	}
 };
 
@@ -249,6 +269,7 @@ static void set_amf_property(amf_base *enc, const wchar_t *name, const T &value)
 /* ------------------------------------------------------------------------- */
 /* Implementation                                                            */
 
+#ifdef _WIN32
 static HMODULE get_lib(const char *lib)
 {
 	HMODULE mod = GetModuleHandleA(lib);
@@ -395,6 +416,7 @@ static void get_tex_from_handle(amf_texencode *enc, uint32_t handle,
 	*km_out = km.Detach();
 	*tex_out = tex.Detach();
 }
+#endif
 
 static constexpr amf_int64 macroblock_size = 16;
 
@@ -640,6 +662,7 @@ static void amf_encode_base(amf_base *enc, AMFSurface *amf_surf,
 static bool amf_encode_tex(void *data, uint32_t handle, int64_t pts,
 			   uint64_t lock_key, uint64_t *next_key,
 			   encoder_packet *packet, bool *received_packet)
+#ifdef _WIN32
 try {
 	amf_texencode *enc = (amf_texencode *)data;
 	ID3D11DeviceContext *context = enc->context;
@@ -716,6 +739,18 @@ try {
 	*received_packet = false;
 	return false;
 }
+#else
+{
+	UNUSED_PARAMETER(data);
+	UNUSED_PARAMETER(handle);
+	UNUSED_PARAMETER(pts);
+	UNUSED_PARAMETER(lock_key);
+	UNUSED_PARAMETER(next_key);
+	UNUSED_PARAMETER(packet);
+	UNUSED_PARAMETER(received_packet);
+	return false;
+}
+#endif
 
 static buf_t alloc_buf(amf_fallback *enc)
 {
@@ -1406,6 +1441,7 @@ static void amf_avc_create_internal(amf_base *enc, obs_data_t *settings)
 
 static void *amf_avc_create_texencode(obs_data_t *settings,
 				      obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::AVC);
 
@@ -1428,6 +1464,12 @@ try {
 	blog(LOG_ERROR, "[texture-amf-h264] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
+}
+#endif
 
 static void *amf_avc_create_fallback(obs_data_t *settings,
 				     obs_encoder_t *encoder)
@@ -1748,6 +1790,7 @@ static void amf_hevc_create_internal(amf_base *enc, obs_data_t *settings)
 
 static void *amf_hevc_create_texencode(obs_data_t *settings,
 				       obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::HEVC);
 
@@ -1770,6 +1813,12 @@ try {
 	blog(LOG_ERROR, "[texture-amf-h265] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
+}
+#endif
 
 static void *amf_hevc_create_fallback(obs_data_t *settings,
 				      obs_encoder_t *encoder)
@@ -2047,6 +2096,7 @@ static void amf_av1_create_internal(amf_base *enc, obs_data_t *settings)
 
 static void *amf_av1_create_texencode(obs_data_t *settings,
 				      obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::AV1);
 
@@ -2069,6 +2119,12 @@ try {
 	blog(LOG_ERROR, "[texture-amf-av1] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
+}
+#endif
 
 static void *amf_av1_create_fallback(obs_data_t *settings,
 				     obs_encoder_t *encoder)
@@ -2151,17 +2207,16 @@ static void register_av1()
 /* ========================================================================= */
 /* Global Stuff                                                              */
 
-static bool enum_luids(void *param, uint32_t idx, uint64_t luid)
-{
-	std::stringstream &cmd = *(std::stringstream *)param;
-	cmd << " " << std::hex << luid;
-	UNUSED_PARAMETER(idx);
-	return true;
-}
+#ifdef _WIN32
+#define OBS_AMF_TEST "obs-amf-test.exe"
+#else
+#define OBS_AMF_TEST "obs-amf-test"
+#endif
 
 extern "C" void amf_load(void)
 try {
 	AMF_RESULT res;
+#ifdef _WIN32
 	HMODULE amf_module_test;
 
 	/* Check if the DLL is present before running the more expensive */
@@ -2171,18 +2226,20 @@ try {
 	if (!amf_module_test)
 		throw "No AMF library";
 	FreeLibrary(amf_module_test);
+#else
+	void *amf_module_test = os_dlopen(AMF_DLL_NAMEA);
+	if (!amf_module_test)
+		throw "No AMF library";
+	os_dlclose(amf_module_test);
+#endif
 
 	/* ----------------------------------- */
 	/* Check for supported codecs          */
 
-	BPtr<char> test_exe = os_get_executable_path_ptr("obs-amf-test.exe");
-	std::stringstream cmd;
+	BPtr<char> test_exe = os_get_executable_path_ptr(OBS_AMF_TEST);
 	std::string caps_str;
 
-	cmd << test_exe;
-	enum_graphics_device_luids(enum_luids, &cmd);
-
-	os_process_pipe_t *pp = os_process_pipe_create(cmd.str().c_str(), "r");
+	os_process_pipe_t *pp = os_process_pipe_create(test_exe, "r");
 	if (!pp)
 		throw "Failed to launch the AMF test process I guess";
 
@@ -2240,12 +2297,12 @@ try {
 	/* ----------------------------------- */
 	/* Init AMF                            */
 
-	amf_module = LoadLibraryW(AMF_DLL_NAME);
+	amf_module = os_dlopen(AMF_DLL_NAMEA);
 	if (!amf_module)
 		throw "AMF library failed to load";
 
 	AMFInit_Fn init =
-		(AMFInit_Fn)GetProcAddress(amf_module, AMF_INIT_FUNCTION_NAME);
+		(AMFInit_Fn)os_dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
 	if (!init)
 		throw "Failed to get AMFInit address";
 
@@ -2257,7 +2314,7 @@ try {
 	if (res != AMF_OK)
 		throw amf_error("GetTrace failed", res);
 
-	AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)GetProcAddress(
+	AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)os_dlsym(
 		amf_module, AMF_QUERY_VERSION_FUNCTION_NAME);
 	if (!get_ver)
 		throw "Failed to get AMFQueryVersion address";
-- 
2.39.1