obs-studio/patches/0006-8832.patch

From a8d13ad4d98e70da648819afc219c65d256cf95d Mon Sep 17 00:00:00 2001
From: David Rosca <nowrep@gmail.com>
Date: Tue, 2 May 2023 13:14:18 +0200
Subject: [PATCH 1/3] libobs: Add AV1 parsing functions

---
 libobs/CMakeLists.txt     |   2 +
 libobs/cmake/legacy.cmake |   2 +
 libobs/obs-av1.c          | 123 ++++++++++++++++++++++++++++++++++++++
 libobs/obs-av1.h          |  35 +++++++++++
 4 files changed, 162 insertions(+)
 create mode 100644 libobs/obs-av1.c
 create mode 100644 libobs/obs-av1.h

diff --git a/libobs/CMakeLists.txt b/libobs/CMakeLists.txt
index 46112abe71f9d..36a75713c001f 100644
--- a/libobs/CMakeLists.txt
+++ b/libobs/CMakeLists.txt
@@ -33,6 +33,8 @@ target_sources(
           obs-audio-controls.c
           obs-audio-controls.h
           obs-audio.c
+          obs-av1.c
+          obs-av1.h
           obs-avc.c
           obs-avc.h
           obs-data.c
diff --git a/libobs/cmake/legacy.cmake b/libobs/cmake/legacy.cmake
index 5f13c4c0f04ab..dd77cdfb94229 100644
--- a/libobs/cmake/legacy.cmake
+++ b/libobs/cmake/legacy.cmake
@@ -32,6 +32,8 @@ target_sources(
           obs-audio.c
           obs-audio-controls.c
           obs-audio-controls.h
+          obs-av1.c
+          obs-av1.h
           obs-avc.c
           obs-avc.h
           obs-data.c
diff --git a/libobs/obs-av1.c b/libobs/obs-av1.c
new file mode 100644
index 0000000000000..a015130537cf2
--- /dev/null
+++ b/libobs/obs-av1.c
@@ -0,0 +1,123 @@
+// SPDX-FileCopyrightText: 2023 David Rosca <nowrep@gmail.com>
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "obs-av1.h"
+
+#include "obs.h"
+
+static inline uint64_t leb128(const uint8_t *buf, size_t size, size_t *len)
+{
+	uint64_t value = 0;
+	uint8_t leb128_byte;
+
+	*len = 0;
+
+	for (int i = 0; i < 8; i++) {
+		if (size-- < 1)
+			break;
+		(*len)++;
+		leb128_byte = buf[i];
+		value |= (leb128_byte & 0x7f) << (i * 7);
+		if (!(leb128_byte & 0x80))
+			break;
+	}
+
+	return value;
+}
+
+static inline unsigned int get_bits(uint8_t val, unsigned int n,
+				    unsigned int count)
+{
+	return (val >> (8 - n - count)) & ((1 << (count - 1)) * 2 - 1);
+}
+
+static void parse_obu_header(const uint8_t *buf, size_t size, size_t *obu_start,
+			     size_t *obu_size, int *obu_type)
+{
+	int extension_flag, has_size_field;
+	size_t size_len = 0;
+
+	*obu_start = 0;
+	*obu_size = 0;
+	*obu_type = 0;
+
+	if (size < 1)
+		return;
+
+	*obu_type = get_bits(*buf, 1, 4);
+	extension_flag = get_bits(*buf, 5, 1);
+	has_size_field = get_bits(*buf, 6, 1);
+
+	if (extension_flag)
+		(*obu_start)++;
+
+	(*obu_start)++;
+
+	if (has_size_field)
+		*obu_size = (size_t)leb128(buf + *obu_start, size - *obu_start,
+					   &size_len);
+	else
+		*obu_size = size - 1;
+
+	*obu_start += size_len;
+}
+
+bool obs_av1_keyframe(const uint8_t *data, size_t size)
+{
+	const uint8_t *start = data, *end = data + size;
+
+	while (start < end) {
+		size_t obu_start, obu_size;
+		int obu_type;
+		parse_obu_header(start, end - start, &obu_start, &obu_size,
+				 &obu_type);
+
+		if (obu_size) {
+			if (obu_type == OBS_OBU_FRAME ||
+			    obu_type == OBS_OBU_FRAME_HEADER) {
+				uint8_t val = *(start + obu_start);
+				if (!get_bits(val, 0, 1)) // show_existing_frame
+					return get_bits(val, 1, 2) ==
+					       0; // frame_type
+				return false;
+			}
+		}
+
+		start += obu_start + obu_size;
+	}
+
+	return false;
+}
+
+void obs_extract_av1_headers(const uint8_t *packet, size_t size,
+			     uint8_t **new_packet_data, size_t *new_packet_size,
+			     uint8_t **header_data, size_t *header_size)
+{
+	DARRAY(uint8_t) new_packet;
+	DARRAY(uint8_t) header;
+	const uint8_t *start = packet, *end = packet + size;
+
+	da_init(new_packet);
+	da_init(header);
+
+	while (start < end) {
+		size_t obu_start, obu_size;
+		int obu_type;
+		parse_obu_header(start, end - start, &obu_start, &obu_size,
+				 &obu_type);
+
+		if (obu_type == OBS_OBU_METADATA ||
+		    obu_type == OBS_OBU_SEQUENCE_HEADER) {
+			da_push_back_array(header, start, obu_start + obu_size);
+		}
+		da_push_back_array(new_packet, start, obu_start + obu_size);
+
+		start += obu_start + obu_size;
+	}
+
+	*new_packet_data = new_packet.array;
+	*new_packet_size = new_packet.num;
+	*header_data = header.array;
+	*header_size = header.num;
+}
diff --git a/libobs/obs-av1.h b/libobs/obs-av1.h
new file mode 100644
index 0000000000000..031299da0a415
--- /dev/null
+++ b/libobs/obs-av1.h
@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: 2023 David Rosca <nowrep@gmail.com>
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "util/c99defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+	OBS_OBU_SEQUENCE_HEADER = 1,
+	OBS_OBU_TEMPORAL_DELIMITER = 2,
+	OBS_OBU_FRAME_HEADER = 3,
+	OBS_OBU_TILE_GROUP = 4,
+	OBS_OBU_METADATA = 5,
+	OBS_OBU_FRAME = 6,
+	OBS_OBU_REDUNDANT_FRAME_HEADER = 7,
+	OBS_OBU_TILE_LIST = 8,
+	OBS_OBU_PADDING = 15,
+};
+
+/* Helpers for parsing AV1 OB units.  */
+
+EXPORT bool obs_av1_keyframe(const uint8_t *data, size_t size);
+EXPORT void obs_extract_av1_headers(const uint8_t *packet, size_t size,
+				    uint8_t **new_packet_data,
+				    size_t *new_packet_size,
+				    uint8_t **header_data, size_t *header_size);
+
+#ifdef __cplusplus
+}
+#endif

From 5d0940dd512515d7885aafdd01cff6a875431418 Mon Sep 17 00:00:00 2001
From: David Rosca <nowrep@gmail.com>
Date: Tue, 2 May 2023 13:39:13 +0200
Subject: [PATCH 2/3] obs-ffmpeg: Add AV1 support for VA-API

---
 plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c | 284 ++++++++++++++++----------
 plugins/obs-ffmpeg/obs-ffmpeg.c       |  19 ++
 plugins/obs-ffmpeg/vaapi-utils.c      |  55 +++++
 plugins/obs-ffmpeg/vaapi-utils.h      |   4 +
 4 files changed, 254 insertions(+), 108 deletions(-)

diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c b/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c
index 25c5f79007a31..7543d68b6f750 100644
--- a/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c
@@ -24,6 +24,7 @@
 #include <media-io/video-io.h>
 #include <obs-module.h>
 #include <obs-avc.h>
+#include <obs-av1.h>
 #ifdef ENABLE_HEVC
 #include <obs-hevc.h>
 #endif
@@ -51,8 +52,15 @@
 #define info(format, ...) do_log(LOG_INFO, format, ##__VA_ARGS__)
 #define debug(format, ...) do_log(LOG_DEBUG, format, ##__VA_ARGS__)
 
+enum codec_type {
+	CODEC_H264,
+	CODEC_HEVC,
+	CODEC_AV1,
+};
+
 struct vaapi_encoder {
 	obs_encoder_t *encoder;
+	enum codec_type codec;
 
 	AVBufferRef *vadevice_ref;
 	AVBufferRef *vaframes_ref;
@@ -83,59 +91,48 @@ static const char *h264_vaapi_getname(void *unused)
 	return "FFmpeg VAAPI H.264";
 }
 
-#ifdef ENABLE_HEVC
-static const char *hevc_vaapi_getname(void *unused)
+static const char *av1_vaapi_getname(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return "FFmpeg VAAPI HEVC";
-}
-#endif
-
-static inline bool h264_valid_format(enum video_format format)
-{
-	return format == VIDEO_FORMAT_NV12;
+	return "FFmpeg VAAPI AV1";
 }
 
 #ifdef ENABLE_HEVC
-static inline bool hevc_valid_format(enum video_format format)
+static const char *hevc_vaapi_getname(void *unused)
 {
-	return (format == VIDEO_FORMAT_NV12) || (format == VIDEO_FORMAT_P010);
+	UNUSED_PARAMETER(unused);
+	return "FFmpeg VAAPI HEVC";
 }
 #endif
 
-static void h264_vaapi_video_info(void *data, struct video_scale_info *info)
+static inline bool vaapi_valid_format(struct vaapi_encoder *enc,
+				      enum video_format format)
 {
-	struct vaapi_encoder *enc = data;
-	enum video_format pref_format;
-
-	pref_format = obs_encoder_get_preferred_video_format(enc->encoder);
-
-	if (!h264_valid_format(pref_format)) {
-		pref_format = h264_valid_format(info->format)
-				      ? info->format
-				      : VIDEO_FORMAT_NV12;
+	if (enc->codec == CODEC_H264) {
+		return format == VIDEO_FORMAT_NV12;
+	} else if (enc->codec == CODEC_HEVC || enc->codec == CODEC_AV1) {
+		return (format == VIDEO_FORMAT_NV12) ||
+		       (format == VIDEO_FORMAT_P010);
+	} else {
+		return false;
 	}
-
-	info->format = pref_format;
 }
 
-#ifdef ENABLE_HEVC
-static void hevc_vaapi_video_info(void *data, struct video_scale_info *info)
+static void vaapi_video_info(void *data, struct video_scale_info *info)
 {
 	struct vaapi_encoder *enc = data;
 	enum video_format pref_format;
 
 	pref_format = obs_encoder_get_preferred_video_format(enc->encoder);
 
-	if (!hevc_valid_format(pref_format)) {
-		pref_format = hevc_valid_format(info->format)
+	if (!vaapi_valid_format(enc, pref_format)) {
+		pref_format = vaapi_valid_format(enc, info->format)
 				      ? info->format
 				      : VIDEO_FORMAT_NV12;
 	}
 
 	info->format = pref_format;
 }
-#endif
 
 static bool vaapi_init_codec(struct vaapi_encoder *enc, const char *path)
 {
@@ -232,7 +229,7 @@ static const rc_mode_t *get_rc_mode(const char *name)
 	return rc_mode ? rc_mode : RC_MODES;
 }
 
-static bool vaapi_update(void *data, obs_data_t *settings, bool hevc)
+static bool vaapi_update(void *data, obs_data_t *settings)
 {
 	struct vaapi_encoder *enc = data;
 
@@ -247,7 +244,7 @@ static bool vaapi_update(void *data, obs_data_t *settings, bool hevc)
 	int bf = (int)obs_data_get_int(settings, "bf");
 	int qp = rc_mode->qp ? (int)obs_data_get_int(settings, "qp") : 0;
 
-	av_opt_set_int(enc->context->priv_data, "qp", qp, 0);
+	enc->context->global_quality = enc->codec == CODEC_AV1 ? qp * 5 : qp;
 
 	int level = (int)obs_data_get_int(settings, "level");
 	int bitrate = rc_mode->bitrate
@@ -277,21 +274,15 @@ static bool vaapi_update(void *data, obs_data_t *settings, bool hevc)
 	info.range = voi->range;
 
 #ifdef ENABLE_HEVC
-	if (hevc) {
+	if (enc->codec == CODEC_HEVC) {
 		if ((profile == FF_PROFILE_HEVC_MAIN) &&
 		    (info.format == VIDEO_FORMAT_P010)) {
 			warn("Forcing Main10 for P010");
 			profile = FF_PROFILE_HEVC_MAIN_10;
 		}
-
-		hevc_vaapi_video_info(enc, &info);
-	} else
-#else
-	UNUSED_PARAMETER(hevc);
-#endif
-	{
-		h264_vaapi_video_info(enc, &info);
 	}
+#endif
+	vaapi_video_info(enc, &info);
 
 	enc->context->profile = profile;
 	enc->context->max_b_frames = bf;
@@ -418,16 +409,28 @@ static void vaapi_destroy(void *data)
 	bfree(enc);
 }
 
+static inline const char *vaapi_encoder_name(enum codec_type codec)
+{
+	if (codec == CODEC_H264) {
+		return "h264_vaapi";
+	} else if (codec == CODEC_HEVC) {
+		return "hevc_vaapi";
+	} else if (codec == CODEC_AV1) {
+		return "av1_vaapi";
+	}
+	return NULL;
+}
+
 static void *vaapi_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
-				   bool hevc)
+				   enum codec_type codec)
 {
 	struct vaapi_encoder *enc;
 
 	enc = bzalloc(sizeof(*enc));
 	enc->encoder = encoder;
 
-	const char *const name = hevc ? "hevc_vaapi" : "h264_vaapi";
-	enc->vaapi = avcodec_find_encoder_by_name(name);
+	enc->codec = codec;
+	enc->vaapi = avcodec_find_encoder_by_name(vaapi_encoder_name(codec));
 
 	enc->first_packet = true;
 
@@ -444,7 +447,7 @@ static void *vaapi_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
 		goto fail;
 	}
 
-	if (!vaapi_update(enc, settings, hevc))
+	if (!vaapi_update(enc, settings))
 		goto fail;
 
 	return enc;
@@ -456,13 +459,18 @@ static void *vaapi_create_internal(obs_data_t *settings, obs_encoder_t *encoder,
 
 static void *h264_vaapi_create(obs_data_t *settings, obs_encoder_t *encoder)
 {
-	return vaapi_create_internal(settings, encoder, false);
+	return vaapi_create_internal(settings, encoder, CODEC_H264);
+}
+
+static void *av1_vaapi_create(obs_data_t *settings, obs_encoder_t *encoder)
+{
+	return vaapi_create_internal(settings, encoder, CODEC_AV1);
 }
 
 #ifdef ENABLE_HEVC
 static void *hevc_vaapi_create(obs_data_t *settings, obs_encoder_t *encoder)
 {
-	return vaapi_create_internal(settings, encoder, true);
+	return vaapi_create_internal(settings, encoder, CODEC_HEVC);
 }
 #endif
 
@@ -492,9 +500,8 @@ static inline void copy_data(AVFrame *pic, const struct encoder_frame *frame,
 	}
 }
 
-static bool vaapi_encode_internal(void *data, struct encoder_frame *frame,
-				  struct encoder_packet *packet,
-				  bool *received_packet, bool hevc)
+static bool vaapi_encode(void *data, struct encoder_frame *frame,
+			 struct encoder_packet *packet, bool *received_packet)
 {
 	struct vaapi_encoder *enc = data;
 	AVFrame *hwframe = NULL;
@@ -556,22 +563,26 @@ static bool vaapi_encode_internal(void *data, struct encoder_frame *frame,
 
 			enc->first_packet = false;
 #ifdef ENABLE_HEVC
-			if (hevc) {
+			if (enc->codec == CODEC_HEVC) {
 				obs_extract_hevc_headers(
 					enc->packet->data, enc->packet->size,
 					&new_packet, &size, &enc->header,
 					&enc->header_size, &enc->sei,
 					&enc->sei_size);
 			} else
-#else
-			UNUSED_PARAMETER(hevc);
 #endif
-			{
+				if (enc->codec == CODEC_H264) {
 				obs_extract_avc_headers(
 					enc->packet->data, enc->packet->size,
 					&new_packet, &size, &enc->header,
 					&enc->header_size, &enc->sei,
 					&enc->sei_size);
+			} else if (enc->codec == CODEC_AV1) {
+				obs_extract_av1_headers(enc->packet->data,
+							enc->packet->size,
+							&new_packet, &size,
+							&enc->header,
+							&enc->header_size);
 			}
 
 			da_copy_array(enc->buffer, new_packet, size);
@@ -587,14 +598,17 @@ static bool vaapi_encode_internal(void *data, struct encoder_frame *frame,
 		packet->size = enc->buffer.num;
 		packet->type = OBS_ENCODER_VIDEO;
 #ifdef ENABLE_HEVC
-		if (hevc) {
+		if (enc->codec == CODEC_HEVC) {
 			packet->keyframe =
 				obs_hevc_keyframe(packet->data, packet->size);
 		} else
 #endif
-		{
+			if (enc->codec == CODEC_H264) {
 			packet->keyframe =
 				obs_avc_keyframe(packet->data, packet->size);
+		} else if (enc->codec == CODEC_AV1) {
+			packet->keyframe =
+				obs_av1_keyframe(packet->data, packet->size);
 		}
 		*received_packet = true;
 	} else {
@@ -610,54 +624,61 @@ static bool vaapi_encode_internal(void *data, struct encoder_frame *frame,
 	return false;
 }
 
-static bool h264_vaapi_encode(void *data, struct encoder_frame *frame,
-			      struct encoder_packet *packet,
-			      bool *received_packet)
+static void set_visible(obs_properties_t *ppts, const char *name, bool visible)
 {
-	return vaapi_encode_internal(data, frame, packet, received_packet,
-				     false);
+	obs_property_t *p = obs_properties_get(ppts, name);
+	obs_property_set_visible(p, visible);
 }
 
-#ifdef ENABLE_HEVC
-static bool hevc_vaapi_encode(void *data, struct encoder_frame *frame,
-			      struct encoder_packet *packet,
-			      bool *received_packet)
+static inline VAProfile vaapi_profile(enum codec_type codec)
 {
-	return vaapi_encode_internal(data, frame, packet, received_packet,
-				     true);
-}
+	if (codec == CODEC_H264) {
+		return VAProfileH264ConstrainedBaseline;
+	} else if (codec == CODEC_AV1) {
+		return VAProfileAV1Profile0;
+#if ENABLE_HEVC
+	} else if (codec == CODEC_HEVC) {
+		return VAProfileHEVCMain;
 #endif
-
-static void set_visible(obs_properties_t *ppts, const char *name, bool visible)
-{
-	obs_property_t *p = obs_properties_get(ppts, name);
-	obs_property_set_visible(p, visible);
+	}
+	return VAProfileNone;
 }
 
-static void vaapi_defaults_internal(obs_data_t *settings, bool hevc)
+static inline const char *vaapi_default_device(enum codec_type codec)
 {
-#ifdef ENABLE_HEVC
-	const char *device = hevc ? vaapi_get_hevc_default_device()
-				  : vaapi_get_h264_default_device();
-#else
-	const char *const device = vaapi_get_h264_default_device();
+	if (codec == CODEC_H264) {
+		return vaapi_get_h264_default_device();
+	} else if (codec == CODEC_AV1) {
+		return vaapi_get_av1_default_device();
+#if ENABLE_HEVC
+	} else if (codec == CODEC_HEVC) {
+		return vaapi_get_hevc_default_device();
 #endif
+	}
+	return NULL;
+}
 
+static void vaapi_defaults_internal(obs_data_t *settings, enum codec_type codec)
+{
+	const char *const device = vaapi_default_device(codec);
 	obs_data_set_default_string(settings, "vaapi_device", device);
 #ifdef ENABLE_HEVC
-	if (hevc) {
+	if (codec == CODEC_HEVC) {
 		obs_data_set_default_int(settings, "profile",
 					 FF_PROFILE_HEVC_MAIN);
+		obs_data_set_default_int(settings, "level", 120);
 
 	} else
-#else
-	UNUSED_PARAMETER(hevc);
 #endif
-	{
+		if (codec == CODEC_H264) {
 		obs_data_set_default_int(settings, "profile",
 					 FF_PROFILE_H264_CONSTRAINED_BASELINE);
+		obs_data_set_default_int(settings, "level", 40);
+	} else if (codec == CODEC_AV1) {
+		obs_data_set_default_int(settings, "profile",
+					 FF_PROFILE_AV1_MAIN);
+		obs_data_set_default_int(settings, "level", 8);
 	}
-	obs_data_set_default_int(settings, "level", 40);
 	obs_data_set_default_int(settings, "bitrate", 2500);
 	obs_data_set_default_int(settings, "keyint_sec", 0);
 	obs_data_set_default_int(settings, "bf", 0);
@@ -670,12 +691,7 @@ static void vaapi_defaults_internal(obs_data_t *settings, bool hevc)
 	if (!va_dpy)
 		return;
 
-#ifdef ENABLE_HEVC
-	const VAProfile profile = hevc ? VAProfileHEVCMain
-				       : VAProfileH264ConstrainedBaseline;
-#else
-	const VAProfile profile = VAProfileH264ConstrainedBaseline;
-#endif
+	const VAProfile profile = vaapi_profile(codec);
 	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_CBR, device))
 		obs_data_set_default_string(settings, "rate_control", "CBR");
 	else if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_VBR, device))
@@ -688,12 +704,17 @@ static void vaapi_defaults_internal(obs_data_t *settings, bool hevc)
 
 static void h264_vaapi_defaults(obs_data_t *settings)
 {
-	vaapi_defaults_internal(settings, false);
+	vaapi_defaults_internal(settings, CODEC_H264);
+}
+
+static void av1_vaapi_defaults(obs_data_t *settings)
+{
+	vaapi_defaults_internal(settings, CODEC_AV1);
 }
 
 static void hevc_vaapi_defaults(obs_data_t *settings)
 {
-	vaapi_defaults_internal(settings, true);
+	vaapi_defaults_internal(settings, CODEC_HEVC);
 }
 
 static bool vaapi_device_modified(obs_properties_t *ppts, obs_property_t *p,
@@ -729,6 +750,11 @@ static bool vaapi_device_modified(obs_properties_t *ppts, obs_property_t *p,
 			goto fail;
 		profile = VAProfileH264High;
 		break;
+	case FF_PROFILE_AV1_MAIN:
+		if (!vaapi_display_av1_supported(va_dpy, device))
+			goto fail;
+		profile = VAProfileAV1Profile0;
+		break;
 #ifdef ENABLE_HEVC
 	case FF_PROFILE_HEVC_MAIN:
 		if (!vaapi_display_hevc_supported(va_dpy, device))
@@ -800,7 +826,7 @@ static bool get_device_name_from_pci(struct pci_access *pacc, char *pci_slot,
 	return false;
 }
 
-static obs_properties_t *vaapi_properties_internal(bool hevc)
+static obs_properties_t *vaapi_properties_internal(enum codec_type codec)
 {
 	obs_properties_t *props = obs_properties_create();
 	obs_property_t *list;
@@ -891,16 +917,18 @@ static obs_properties_t *vaapi_properties_internal(bool hevc)
 				       obs_module_text("Profile"),
 				       OBS_COMBO_TYPE_LIST,
 				       OBS_COMBO_FORMAT_INT);
-	if (hevc) {
+	if (codec == CODEC_HEVC) {
 		obs_property_list_add_int(list, "Main", FF_PROFILE_HEVC_MAIN);
 		obs_property_list_add_int(list, "Main10",
 					  FF_PROFILE_HEVC_MAIN_10);
-	} else {
+	} else if (codec == CODEC_H264) {
 		obs_property_list_add_int(list,
 					  "Constrained Baseline (default)",
 					  FF_PROFILE_H264_CONSTRAINED_BASELINE);
 		obs_property_list_add_int(list, "Main", FF_PROFILE_H264_MAIN);
 		obs_property_list_add_int(list, "High", FF_PROFILE_H264_HIGH);
+	} else if (codec == CODEC_AV1) {
+		obs_property_list_add_int(list, "Main", FF_PROFILE_AV1_MAIN);
 	}
 
 	obs_property_set_modified_callback(list, vaapi_device_modified);
@@ -909,15 +937,34 @@ static obs_properties_t *vaapi_properties_internal(bool hevc)
 				       OBS_COMBO_TYPE_LIST,
 				       OBS_COMBO_FORMAT_INT);
 	obs_property_list_add_int(list, "Auto", FF_LEVEL_UNKNOWN);
-	obs_property_list_add_int(list, "3.0", 30);
-	obs_property_list_add_int(list, "3.1", 31);
-	obs_property_list_add_int(list, "4.0 (default) (Compatibility mode)",
-				  40);
-	obs_property_list_add_int(list, "4.1", 41);
-	obs_property_list_add_int(list, "4.2", 42);
-	obs_property_list_add_int(list, "5.0", 50);
-	obs_property_list_add_int(list, "5.1", 51);
-	obs_property_list_add_int(list, "5.2", 52);
+	if (codec == CODEC_H264) {
+		obs_property_list_add_int(list, "3.0", 30);
+		obs_property_list_add_int(list, "3.1", 31);
+		obs_property_list_add_int(
+			list, "4.0 (default) (Compatibility mode)", 40);
+		obs_property_list_add_int(list, "4.1", 41);
+		obs_property_list_add_int(list, "4.2", 42);
+		obs_property_list_add_int(list, "5.0", 50);
+		obs_property_list_add_int(list, "5.1", 51);
+		obs_property_list_add_int(list, "5.2", 52);
+	} else if (codec == CODEC_HEVC) {
+		obs_property_list_add_int(list, "3.0", 90);
+		obs_property_list_add_int(list, "3.1", 93);
+		obs_property_list_add_int(list, "4.0 (default)", 120);
+		obs_property_list_add_int(list, "4.1", 123);
+		obs_property_list_add_int(list, "5.0", 150);
+		obs_property_list_add_int(list, "5.1", 153);
+		obs_property_list_add_int(list, "5.2", 156);
+	} else if (codec == CODEC_AV1) {
+		obs_property_list_add_int(list, "3.0", 4);
+		obs_property_list_add_int(list, "3.1", 5);
+		obs_property_list_add_int(list, "4.0 (default)", 8);
+		obs_property_list_add_int(list, "4.1", 9);
+		obs_property_list_add_int(list, "5.0", 12);
+		obs_property_list_add_int(list, "5.1", 13);
+		obs_property_list_add_int(list, "5.2", 14);
+		obs_property_list_add_int(list, "5.3", 15);
+	}
 
 	list = obs_properties_add_list(props, "rate_control",
 				       obs_module_text("RateControl"),
@@ -952,14 +999,20 @@ static obs_properties_t *vaapi_properties_internal(bool hevc)
 static obs_properties_t *h264_vaapi_properties(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return vaapi_properties_internal(false);
+	return vaapi_properties_internal(CODEC_H264);
+}
+
+static obs_properties_t *av1_vaapi_properties(void *unused)
+{
+	UNUSED_PARAMETER(unused);
+	return vaapi_properties_internal(CODEC_AV1);
 }
 
 #ifdef ENABLE_HEVC
 static obs_properties_t *hevc_vaapi_properties(void *unused)
 {
 	UNUSED_PARAMETER(unused);
-	return vaapi_properties_internal(true);
+	return vaapi_properties_internal(CODEC_HEVC);
 }
 #endif
 
@@ -988,12 +1041,27 @@ struct obs_encoder_info h264_vaapi_encoder_info = {
 	.get_name = h264_vaapi_getname,
 	.create = h264_vaapi_create,
 	.destroy = vaapi_destroy,
-	.encode = h264_vaapi_encode,
+	.encode = vaapi_encode,
 	.get_defaults = h264_vaapi_defaults,
 	.get_properties = h264_vaapi_properties,
 	.get_extra_data = vaapi_extra_data,
 	.get_sei_data = vaapi_sei_data,
-	.get_video_info = h264_vaapi_video_info,
+	.get_video_info = vaapi_video_info,
+};
+
+struct obs_encoder_info av1_vaapi_encoder_info = {
+	.id = "av1_ffmpeg_vaapi",
+	.type = OBS_ENCODER_VIDEO,
+	.codec = "av1",
+	.get_name = av1_vaapi_getname,
+	.create = av1_vaapi_create,
+	.destroy = vaapi_destroy,
+	.encode = vaapi_encode,
+	.get_defaults = av1_vaapi_defaults,
+	.get_properties = av1_vaapi_properties,
+	.get_extra_data = vaapi_extra_data,
+	.get_sei_data = vaapi_sei_data,
+	.get_video_info = vaapi_video_info,
 };
 
 #ifdef ENABLE_HEVC
@@ -1004,11 +1072,11 @@ struct obs_encoder_info hevc_vaapi_encoder_info = {
 	.get_name = hevc_vaapi_getname,
 	.create = hevc_vaapi_create,
 	.destroy = vaapi_destroy,
-	.encode = hevc_vaapi_encode,
+	.encode = vaapi_encode,
 	.get_defaults = hevc_vaapi_defaults,
 	.get_properties = hevc_vaapi_properties,
 	.get_extra_data = vaapi_extra_data,
 	.get_sei_data = vaapi_sei_data,
-	.get_video_info = hevc_vaapi_video_info,
+	.get_video_info = vaapi_video_info,
 };
 #endif
diff --git a/plugins/obs-ffmpeg/obs-ffmpeg.c b/plugins/obs-ffmpeg/obs-ffmpeg.c
index 7eb9a876d82ff..981bb9c6550ff 100644
--- a/plugins/obs-ffmpeg/obs-ffmpeg.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg.c
@@ -47,6 +47,7 @@ extern struct obs_encoder_info aom_av1_encoder_info;
 
 #ifdef LIBAVUTIL_VAAPI_AVAILABLE
 extern struct obs_encoder_info h264_vaapi_encoder_info;
+extern struct obs_encoder_info av1_vaapi_encoder_info;
 #ifdef ENABLE_HEVC
 extern struct obs_encoder_info hevc_vaapi_encoder_info;
 #endif
@@ -337,6 +338,17 @@ static bool h264_vaapi_supported(void)
 	 * that support H264. */
 	return vaapi_get_h264_default_device() != NULL;
 }
+static bool av1_vaapi_supported(void)
+{
+	const AVCodec *vaenc = avcodec_find_encoder_by_name("av1_vaapi");
+
+	if (!vaenc)
+		return false;
+
+	/* NOTE: If default device is NULL, it means there is no device
+	 * that support AV1. */
+	return vaapi_get_av1_default_device() != NULL;
+}
 #ifdef ENABLE_HEVC
 static bool hevc_vaapi_supported(void)
 {
@@ -447,6 +459,13 @@ bool obs_module_load(void)
 		blog(LOG_INFO, "FFmpeg VAAPI H264 encoding not supported");
 	}
 
+	if (av1_vaapi_supported()) {
+		blog(LOG_INFO, "FFmpeg VAAPI AV1 encoding supported");
+		obs_register_encoder(&av1_vaapi_encoder_info);
+	} else {
+		blog(LOG_INFO, "FFmpeg VAAPI AV1 encoding not supported");
+	}
+
 #ifdef ENABLE_HEVC
 	if (hevc_vaapi_supported()) {
 		blog(LOG_INFO, "FFmpeg VAAPI HEVC encoding supported");
diff --git a/plugins/obs-ffmpeg/vaapi-utils.c b/plugins/obs-ffmpeg/vaapi-utils.c
index 4198ee59c6ab4..2c8eddd3676fa 100644
--- a/plugins/obs-ffmpeg/vaapi-utils.c
+++ b/plugins/obs-ffmpeg/vaapi-utils.c
@@ -261,6 +261,61 @@ const char *vaapi_get_h264_default_device()
 	return default_h264_device;
 }
 
+bool vaapi_display_av1_supported(VADisplay dpy, const char *device_path)
+{
+	bool ret = false;
+
+	CHECK_PROFILE(ret, VAProfileAV1Profile0, dpy, device_path);
+
+	if (!ret) {
+		CHECK_PROFILE_LP(ret, VAProfileAV1Profile0, dpy, device_path);
+	}
+
+	return ret;
+}
+
+bool vaapi_device_av1_supported(const char *device_path)
+{
+	bool ret = false;
+	VADisplay va_dpy;
+
+	int drm_fd = -1;
+
+	va_dpy = vaapi_open_device(&drm_fd, device_path,
+				   "vaapi_device_av1_supported");
+	if (!va_dpy)
+		return false;
+
+	ret = vaapi_display_av1_supported(va_dpy, device_path);
+
+	vaapi_close_device(&drm_fd, va_dpy);
+
+	return ret;
+}
+
+const char *vaapi_get_av1_default_device()
+{
+	static const char *default_av1_device = NULL;
+
+	if (!default_av1_device) {
+		bool ret = false;
+		char path[32] = "/dev/dri/renderD1";
+		for (int i = 28;; i++) {
+			sprintf(path, "/dev/dri/renderD1%d", i);
+			if (access(path, F_OK) != 0)
+				break;
+
+			ret = vaapi_device_av1_supported(path);
+			if (ret) {
+				default_av1_device = strdup(path);
+				break;
+			}
+		}
+	}
+
+	return default_av1_device;
+}
+
 #ifdef ENABLE_HEVC
 
 bool vaapi_display_hevc_supported(VADisplay dpy, const char *device_path)
diff --git a/plugins/obs-ffmpeg/vaapi-utils.h b/plugins/obs-ffmpeg/vaapi-utils.h
index b90b2394710b3..2d7ee29dc9ab9 100644
--- a/plugins/obs-ffmpeg/vaapi-utils.h
+++ b/plugins/obs-ffmpeg/vaapi-utils.h
@@ -19,6 +19,10 @@ bool vaapi_display_h264_supported(VADisplay dpy, const char *device_path);
 bool vaapi_device_h264_supported(const char *device_path);
 const char *vaapi_get_h264_default_device(void);
 
+bool vaapi_display_av1_supported(VADisplay dpy, const char *device_path);
+bool vaapi_device_av1_supported(const char *device_path);
+const char *vaapi_get_av1_default_device(void);
+
 #ifdef ENABLE_HEVC
 bool vaapi_display_hevc_supported(VADisplay dpy, const char *device_path);
 bool vaapi_device_hevc_supported(const char *device_path);

From 1b2b4da94760c829366e129cddda7cfe886f7b48 Mon Sep 17 00:00:00 2001
From: David Rosca <nowrep@gmail.com>
Date: Fri, 22 Sep 2023 08:11:46 +0200
Subject: [PATCH 3/3] obs-ffmpeg: Set better VA-API defaults

Use High profile for H264 and auto level with all codecs.
Remove setting default value for unused "rendermode" option.
---
 plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c | 29 ++++++++++-----------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c b/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c
index 7543d68b6f750..0ec7fa2ec43d0 100644
--- a/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c
+++ b/plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c
@@ -663,26 +663,21 @@ static void vaapi_defaults_internal(obs_data_t *settings, enum codec_type codec)
 	const char *const device = vaapi_default_device(codec);
 	obs_data_set_default_string(settings, "vaapi_device", device);
 #ifdef ENABLE_HEVC
-	if (codec == CODEC_HEVC) {
+	if (codec == CODEC_HEVC)
 		obs_data_set_default_int(settings, "profile",
 					 FF_PROFILE_HEVC_MAIN);
-		obs_data_set_default_int(settings, "level", 120);
-
-	} else
+	else
 #endif
-		if (codec == CODEC_H264) {
+		if (codec == CODEC_H264)
 		obs_data_set_default_int(settings, "profile",
-					 FF_PROFILE_H264_CONSTRAINED_BASELINE);
-		obs_data_set_default_int(settings, "level", 40);
-	} else if (codec == CODEC_AV1) {
+					 FF_PROFILE_H264_HIGH);
+	else if (codec == CODEC_AV1)
 		obs_data_set_default_int(settings, "profile",
 					 FF_PROFILE_AV1_MAIN);
-		obs_data_set_default_int(settings, "level", 8);
-	}
+	obs_data_set_default_int(settings, "level", FF_LEVEL_UNKNOWN);
 	obs_data_set_default_int(settings, "bitrate", 2500);
 	obs_data_set_default_int(settings, "keyint_sec", 0);
 	obs_data_set_default_int(settings, "bf", 0);
-	obs_data_set_default_int(settings, "rendermode", 0);
 	obs_data_set_default_int(settings, "qp", 20);
 	obs_data_set_default_int(settings, "maxrate", 0);
 
@@ -770,7 +765,7 @@ static bool vaapi_device_modified(obs_properties_t *ppts, obs_property_t *p,
 	}
 
 	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_CBR, device))
-		obs_property_list_add_string(rc_p, "CBR (default)", "CBR");
+		obs_property_list_add_string(rc_p, "CBR", "CBR");
 
 	if (vaapi_device_rc_supported(profile, va_dpy, VA_RC_VBR, device))
 		obs_property_list_add_string(rc_p, "VBR", "VBR");
@@ -922,8 +917,7 @@ static obs_properties_t *vaapi_properties_internal(enum codec_type codec)
 		obs_property_list_add_int(list, "Main10",
 					  FF_PROFILE_HEVC_MAIN_10);
 	} else if (codec == CODEC_H264) {
-		obs_property_list_add_int(list,
-					  "Constrained Baseline (default)",
+		obs_property_list_add_int(list, "Constrained Baseline",
 					  FF_PROFILE_H264_CONSTRAINED_BASELINE);
 		obs_property_list_add_int(list, "Main", FF_PROFILE_H264_MAIN);
 		obs_property_list_add_int(list, "High", FF_PROFILE_H264_HIGH);
@@ -940,8 +934,7 @@ static obs_properties_t *vaapi_properties_internal(enum codec_type codec)
 	if (codec == CODEC_H264) {
 		obs_property_list_add_int(list, "3.0", 30);
 		obs_property_list_add_int(list, "3.1", 31);
-		obs_property_list_add_int(
-			list, "4.0 (default) (Compatibility mode)", 40);
+		obs_property_list_add_int(list, "4.0", 40);
 		obs_property_list_add_int(list, "4.1", 41);
 		obs_property_list_add_int(list, "4.2", 42);
 		obs_property_list_add_int(list, "5.0", 50);
@@ -950,7 +943,7 @@ static obs_properties_t *vaapi_properties_internal(enum codec_type codec)
 	} else if (codec == CODEC_HEVC) {
 		obs_property_list_add_int(list, "3.0", 90);
 		obs_property_list_add_int(list, "3.1", 93);
-		obs_property_list_add_int(list, "4.0 (default)", 120);
+		obs_property_list_add_int(list, "4.0", 120);
 		obs_property_list_add_int(list, "4.1", 123);
 		obs_property_list_add_int(list, "5.0", 150);
 		obs_property_list_add_int(list, "5.1", 153);
@@ -958,7 +951,7 @@ static obs_properties_t *vaapi_properties_internal(enum codec_type codec)
 	} else if (codec == CODEC_AV1) {
 		obs_property_list_add_int(list, "3.0", 4);
 		obs_property_list_add_int(list, "3.1", 5);
-		obs_property_list_add_int(list, "4.0 (default)", 8);
+		obs_property_list_add_int(list, "4.0", 8);
 		obs_property_list_add_int(list, "4.1", 9);
 		obs_property_list_add_int(list, "5.0", 12);
 		obs_property_list_add_int(list, "5.1", 13);