obs-studio/patches/7206.patch

diff '--color=auto' -ru a/plugins/obs-ffmpeg/CMakeLists.txt b/plugins/obs-ffmpeg/CMakeLists.txt
--- a/plugins/obs-ffmpeg/CMakeLists.txt	2023-02-04 13:17:10.000000000 +0300
+++ b/plugins/obs-ffmpeg/CMakeLists.txt	2023-03-04 21:01:11.132594927 +0300
@@ -119,11 +119,14 @@
             obs-ffmpeg.rc)

 elseif(OS_POSIX AND NOT OS_MACOS)
+  add_subdirectory(obs-amf-test)
   find_package(Libva REQUIRED)
   find_package(Libpci REQUIRED)
   target_sources(obs-ffmpeg PRIVATE obs-ffmpeg-vaapi.c vaapi-utils.c
                                     vaapi-utils.h)
   target_link_libraries(obs-ffmpeg PRIVATE Libva::va Libva::drm LIBPCI::LIBPCI)
+  target_sources(obs-ffmpeg PRIVATE texture-amf.cpp)
+  target_link_libraries(obs-ffmpeg PRIVATE LIBPCI::LIBPCI)
 endif()

 setup_plugin_target(obs-ffmpeg)
diff '--color=auto' -ru a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt
--- a/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt	2023-02-04 13:17:10.000000000 +0300
+++ b/plugins/obs-ffmpeg/obs-amf-test/CMakeLists.txt	2023-03-04 20:58:48.104742328 +0300
@@ -3,8 +3,15 @@
 include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/libobs)

 add_executable(obs-amf-test)
-target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
-target_link_libraries(obs-amf-test d3d11 dxgi dxguid)
+
+if(OS_WINDOWS)
+  target_sources(obs-amf-test PRIVATE obs-amf-test.cpp)
+  target_link_libraries(obs-amf-test d3d11 dxgi dxguid)
+elseif(OS_POSIX AND NOT OS_MACOS)
+  find_package(Vulkan REQUIRED)
+  target_sources(obs-amf-test PRIVATE obs-amf-test-linux.cpp)
+  target_link_libraries(obs-amf-test dl Vulkan::Vulkan)
+endif()

 set_target_properties(obs-amf-test PROPERTIES FOLDER "plugins/obs-ffmpeg")

Only in b/plugins/obs-ffmpeg/obs-amf-test: obs-amf-test-linux.cpp
diff '--color=auto' -ru a/plugins/obs-ffmpeg/obs-ffmpeg.c b/plugins/obs-ffmpeg/obs-ffmpeg.c
--- a/plugins/obs-ffmpeg/obs-ffmpeg.c	2023-02-04 13:17:10.000000000 +0300
+++ b/plugins/obs-ffmpeg/obs-ffmpeg.c	2023-03-04 20:58:48.104742328 +0300
@@ -339,6 +339,9 @@
 #ifdef _WIN32
 extern void jim_nvenc_load(bool h264, bool hevc, bool av1);
 extern void jim_nvenc_unload(void);
+#endif
+
+#if defined(_WIN32) || defined(__linux__)
 extern void amf_load(void);
 extern void amf_unload(void);
 #endif
@@ -408,7 +411,7 @@
 #endif
 	}

-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
 	amf_load();
 #endif

@@ -440,8 +443,11 @@
 	obs_ffmpeg_unload_logging();
 #endif

-#ifdef _WIN32
+#if defined(_WIN32) || defined(__linux__)
 	amf_unload();
+#endif
+
+#ifdef _WIN32
 	jim_nvenc_unload();
 #endif
 }
Only in b/plugins/obs-ffmpeg: obs-ffmpeg.c.orig
diff '--color=auto' -ru a/plugins/obs-ffmpeg/texture-amf.cpp b/plugins/obs-ffmpeg/texture-amf.cpp
--- a/plugins/obs-ffmpeg/texture-amf.cpp	2023-02-04 13:17:10.000000000 +0300
+++ b/plugins/obs-ffmpeg/texture-amf.cpp	2023-03-04 20:58:48.108075702 +0300
@@ -20,6 +20,7 @@
 #include "external/AMF/include/core/Factory.h"
 #include "external/AMF/include/core/Trace.h"

+#ifdef _WIN32
 #include <dxgi.h>
 #include <d3d11.h>
 #include <d3d11_1.h>
@@ -27,6 +28,8 @@
 #include <util/windows/device-enum.h>
 #include <util/windows/HRError.hpp>
 #include <util/windows/ComPtr.hpp>
+#endif
+
 #include <util/platform.h>
 #include <util/util.hpp>
 #include <util/pipe.h>
@@ -57,8 +60,10 @@

 struct handle_tex {
 	uint32_t handle;
+#ifdef _WIN32
 	ComPtr<ID3D11Texture2D> tex;
 	ComPtr<IDXGIKeyedMutex> km;
+#endif
 };

 struct adapter_caps {
@@ -74,7 +79,7 @@
 static bool h264_supported = false;
 static AMFFactory *amf_factory = nullptr;
 static AMFTrace *amf_trace = nullptr;
-static HMODULE amf_module = nullptr;
+static void *amf_module = nullptr;
 static uint64_t amf_version = 0;

 /* ========================================================================= */
@@ -122,9 +127,11 @@
 	virtual void init() = 0;
 };

-using d3dtex_t = ComPtr<ID3D11Texture2D>;
 using buf_t = std::vector<uint8_t>;

+#ifdef _WIN32
+using d3dtex_t = ComPtr<ID3D11Texture2D>;
+
 struct amf_texencode : amf_base, public AMFSurfaceObserver {
 	volatile bool destroying = false;

@@ -161,6 +168,7 @@
 			throw amf_error("InitDX11 failed", res);
 	}
 };
+#endif

 struct amf_fallback : amf_base, public AMFSurfaceObserver {
 	volatile bool destroying = false;
@@ -188,9 +196,21 @@

 	void init() override
 	{
+#if defined(_WIN32)
 		AMF_RESULT res = amf_context->InitDX11(nullptr, AMF_DX11_1);
 		if (res != AMF_OK)
 			throw amf_error("InitDX11 failed", res);
+#elif defined(__linux__)
+		AMFContext1 *context1 = NULL;
+		AMF_RESULT res = amf_context->QueryInterface(
+			AMFContext1::IID(), (void **)&context1);
+		if (res != AMF_OK)
+			throw amf_error("CreateContext1 failed", res);
+		res = context1->InitVulkan(nullptr);
+		context1->Release();
+		if (res != AMF_OK)
+			throw amf_error("InitVulkan failed", res);
+#endif
 	}
 };

@@ -249,6 +269,7 @@
 /* ------------------------------------------------------------------------- */
 /* Implementation                                                            */

+#ifdef _WIN32
 static HMODULE get_lib(const char *lib)
 {
 	HMODULE mod = GetModuleHandleA(lib);
@@ -395,6 +416,7 @@
 	*km_out = km.Detach();
 	*tex_out = tex.Detach();
 }
+#endif

 static constexpr amf_int64 macroblock_size = 16;

@@ -640,6 +662,7 @@
 static bool amf_encode_tex(void *data, uint32_t handle, int64_t pts,
 			   uint64_t lock_key, uint64_t *next_key,
 			   encoder_packet *packet, bool *received_packet)
+#ifdef _WIN32
 try {
 	amf_texencode *enc = (amf_texencode *)data;
 	ID3D11DeviceContext *context = enc->context;
@@ -716,6 +739,18 @@
 	*received_packet = false;
 	return false;
 }
+#else
+{
+	UNUSED_PARAMETER(data);
+	UNUSED_PARAMETER(handle);
+	UNUSED_PARAMETER(pts);
+	UNUSED_PARAMETER(lock_key);
+	UNUSED_PARAMETER(next_key);
+	UNUSED_PARAMETER(packet);
+	UNUSED_PARAMETER(received_packet);
+	return false;
+}
+#endif

 static buf_t alloc_buf(amf_fallback *enc)
 {
@@ -1406,6 +1441,7 @@

 static void *amf_avc_create_texencode(obs_data_t *settings,
 				      obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::AVC);

@@ -1428,6 +1464,12 @@
 	blog(LOG_ERROR, "[texture-amf-h264] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "h264_fallback_amf");
+}
+#endif

 static void *amf_avc_create_fallback(obs_data_t *settings,
 				     obs_encoder_t *encoder)
@@ -1748,6 +1790,7 @@

 static void *amf_hevc_create_texencode(obs_data_t *settings,
 				       obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::HEVC);

@@ -1770,6 +1813,12 @@
 	blog(LOG_ERROR, "[texture-amf-h265] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "h265_fallback_amf");
+}
+#endif

 static void *amf_hevc_create_fallback(obs_data_t *settings,
 				      obs_encoder_t *encoder)
@@ -2047,6 +2096,7 @@

 static void *amf_av1_create_texencode(obs_data_t *settings,
 				      obs_encoder_t *encoder)
+#ifdef _WIN32
 try {
 	check_texture_encode_capability(encoder, amf_codec_type::AV1);

@@ -2069,6 +2119,12 @@
 	blog(LOG_ERROR, "[texture-amf-av1] %s: %s", __FUNCTION__, err);
 	return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
 }
+#else
+{
+	UNUSED_PARAMETER(settings);
+	return obs_encoder_create_rerouted(encoder, "av1_fallback_amf");
+}
+#endif

 static void *amf_av1_create_fallback(obs_data_t *settings,
 				     obs_encoder_t *encoder)
@@ -2159,9 +2215,16 @@
 	return true;
 }

+#ifdef _WIN32
+#define OBS_AMF_TEST "obs-amf-test.exe"
+#else
+#define OBS_AMF_TEST "obs-amf-test"
+#endif
+
 extern "C" void amf_load(void)
 try {
 	AMF_RESULT res;
+#ifdef _WIN32
 	HMODULE amf_module_test;

 	/* Check if the DLL is present before running the more expensive */
@@ -2171,17 +2234,25 @@
 	if (!amf_module_test)
 		throw "No AMF library";
 	FreeLibrary(amf_module_test);
+#else
+	void *amf_module_test = os_dlopen(AMF_DLL_NAMEA);
+	if (!amf_module_test)
+		throw "No AMF library";
+	os_dlclose(amf_module_test);
+#endif

 	/* ----------------------------------- */
 	/* Check for supported codecs          */

-	BPtr<char> test_exe = os_get_executable_path_ptr("obs-amf-test.exe");
+	BPtr<char> test_exe = os_get_executable_path_ptr(OBS_AMF_TEST);
 	std::stringstream cmd;
 	std::string caps_str;

 	cmd << test_exe;
-	enum_graphics_device_luids(enum_luids, &cmd);

+#ifdef _WIN32
+	enum_graphics_device_luids(enum_luids, &cmd);
+#endif
 	os_process_pipe_t *pp = os_process_pipe_create(cmd.str().c_str(), "r");
 	if (!pp)
 		throw "Failed to launch the AMF test process I guess";
@@ -2240,12 +2311,12 @@
 	/* ----------------------------------- */
 	/* Init AMF                            */

-	amf_module = LoadLibraryW(AMF_DLL_NAME);
+	amf_module = os_dlopen(AMF_DLL_NAMEA);
 	if (!amf_module)
 		throw "AMF library failed to load";

 	AMFInit_Fn init =
-		(AMFInit_Fn)GetProcAddress(amf_module, AMF_INIT_FUNCTION_NAME);
+		(AMFInit_Fn)os_dlsym(amf_module, AMF_INIT_FUNCTION_NAME);
 	if (!init)
 		throw "Failed to get AMFInit address";

@@ -2257,7 +2328,7 @@
 	if (res != AMF_OK)
 		throw amf_error("GetTrace failed", res);

-	AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)GetProcAddress(
+	AMFQueryVersion_Fn get_ver = (AMFQueryVersion_Fn)os_dlsym(
 		amf_module, AMF_QUERY_VERSION_FUNCTION_NAME);
 	if (!get_ver)
 		throw "Failed to get AMFQueryVersion address";
diff '--color=auto' -ru a/plugins/obs-ffmpeg/texture-amf-opts.hpp b/plugins/obs-ffmpeg/texture-amf-opts.hpp
--- a/plugins/obs-ffmpeg/texture-amf-opts.hpp	2023-02-04 13:17:10.000000000 +0300
+++ b/plugins/obs-ffmpeg/texture-amf-opts.hpp	2023-03-04 20:58:48.104742328 +0300
@@ -321,7 +321,7 @@
 			val = atoi(opt->value);
 		}

-		os_utf8_to_wcs(opt->name, 0, wname, _countof(wname));
+		os_utf8_to_wcs(opt->name, 0, wname, amf_countof(wname));
 		if (is_bool) {
 			bool bool_val = (bool)val;
 			set_amf_property(enc, wname, bool_val);