From 9d53d773fa9010b4a753997b22df65dbd95d21bf Mon Sep 17 00:00:00 2001 From: Yury Date: Sat, 24 Dec 2022 14:00:06 +0200 Subject: [PATCH] Entropy coding option (#1318) * Add entropy coding option * Use CAVLC as default option * Update description for entropy options * Add entropy coding option to Linux * Fix indentation * Add missing break statements --- alvr/dashboard/js/app/nls/settings.js | 5 +++ alvr/server/cpp/ALVR-common/packet_types.h | 5 +++ alvr/server/cpp/alvr_server/Settings.cpp | 1 + alvr/server/cpp/alvr_server/Settings.h | 1 + .../platform/linux/EncodePipelineNvEnc.cpp | 10 +++++ .../platform/linux/EncodePipelineVAAPI.cpp | 11 +++++- .../cpp/platform/win32/VideoEncoderNVENC.cpp | 9 +++++ .../cpp/platform/win32/VideoEncoderSW.cpp | 10 ++++- .../cpp/platform/win32/VideoEncoderVCE.cpp | 38 ++++++++++++------- alvr/server/src/connection.rs | 1 + alvr/session/src/lib.rs | 1 + alvr/session/src/settings.rs | 14 +++++++ 12 files changed, 90 insertions(+), 16 deletions(-) diff --git a/alvr/dashboard/js/app/nls/settings.js b/alvr/dashboard/js/app/nls/settings.js index 9cf4275f74..a849d53f01 100644 --- a/alvr/dashboard/js/app/nls/settings.js +++ b/alvr/dashboard/js/app/nls/settings.js @@ -80,6 +80,11 @@ define({ "Specifies the bitrate control method (Windows with AMD and nVidia cards only). CBR - constant bitrate, VBR - variable bitrate.", "_root_video_rateControlMode_cbr-choice-.name": "CBR", "_root_video_rateControlMode_vbr-choice-.name": "VBR", + "_root_video_entropyCoding-choice-.name": "Entropy coding method", + "_root_video_entropyCoding-choice-.description": + "Specifies the entropy coding method (h264 only). CABAC - provides better quality at lower bitrate, but may increase the encoder/decoder latency, CAVLC - provides worse quality that can be compensated with higher bitrate, and may significantly decrease encoder/decoder latency.", + "_root_video_entropyCoding_cabac-choice-.name": "CABAC", + "_root_video_entropyCoding_cavlc-choice-.name": "CAVLC", "_root_video_advancedCodecOptions_amfControls_usePreproc.name": "Enable Pre-Processor component", "_root_video_advancedCodecOptions_amfControls_usePreproc.description": diff --git a/alvr/server/cpp/ALVR-common/packet_types.h b/alvr/server/cpp/ALVR-common/packet_types.h index 4211be4b0e..ef45da7660 100644 --- a/alvr/server/cpp/ALVR-common/packet_types.h +++ b/alvr/server/cpp/ALVR-common/packet_types.h @@ -23,6 +23,11 @@ enum ALVR_RATE_CONTROL_METHOD { ALVR_VBR = 1, }; +enum ALVR_ENTROPY_CODING { + ALVR_CABAC = 0, + ALVR_CAVLC = 1, +}; + enum ALVR_INPUT { ALVR_INPUT_SYSTEM_CLICK, ALVR_INPUT_APPLICATION_MENU_CLICK, diff --git a/alvr/server/cpp/alvr_server/Settings.cpp b/alvr/server/cpp/alvr_server/Settings.cpp index f2dc67f63c..4cbe90a69c 100644 --- a/alvr/server/cpp/alvr_server/Settings.cpp +++ b/alvr/server/cpp/alvr_server/Settings.cpp @@ -86,6 +86,7 @@ void Settings::Load() m_codec = (int32_t)config.get("codec").get(); m_rateControlMode = (uint32_t)config.get("rate_control_mode").get(); + m_entropyCoding = (uint32_t)config.get("entropy_coding").get(); m_refreshRate = (int)config.get("refresh_rate").get(); mEncodeBitrateMBs = (int)config.get("encode_bitrate_mbs").get(); m_enableAdaptiveBitrate = config.get("enable_adaptive_bitrate").get(); diff --git a/alvr/server/cpp/alvr_server/Settings.h b/alvr/server/cpp/alvr_server/Settings.h index 04433b3bd3..4d2e3bef68 100644 --- a/alvr/server/cpp/alvr_server/Settings.h +++ b/alvr/server/cpp/alvr_server/Settings.h @@ -79,6 +79,7 @@ class Settings uint32_t m_preProcTor; uint32_t m_encoderQualityPreset; uint32_t m_rateControlMode; + uint32_t m_entropyCoding; bool m_force_sw_encoding; uint32_t m_swThreadCount; diff --git a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp index 01ad5480af..e9d1e5c40d 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineNvEnc.cpp @@ -51,6 +51,16 @@ alvr::EncodePipelineNvEnc::EncodePipelineNvEnc(std::vector &input_frame case ALVR_CODEC_H264: AVUTIL.av_opt_set(encoder_ctx, "preset", "llhq", 0); AVUTIL.av_opt_set(encoder_ctx, "zerolatency", "1", 0); + + switch (settings.m_entropyCoding) { + case ALVR_CABAC: + AVUTIL.av_opt_set(encoder_ctx->priv_data, "coder", "ac", 0); + break; + case ALVR_CAVLC: + AVUTIL.av_opt_set(encoder_ctx->priv_data, "coder", "vlc", 0); + break; + } + break; case ALVR_CODEC_H265: AVUTIL.av_opt_set(encoder_ctx, "preset", "llhq", 0); diff --git a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp index 9550bdf2ba..4d8bc2a004 100644 --- a/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp +++ b/alvr/server/cpp/platform/linux/EncodePipelineVAAPI.cpp @@ -130,7 +130,16 @@ alvr::EncodePipelineVAAPI::EncodePipelineVAAPI(std::vector& input_frame { case ALVR_CODEC_H264: encoder_ctx->profile = FF_PROFILE_H264_MAIN; - AVUTIL.av_opt_set(encoder_ctx, "rc_mode", "2", 0); //CBR + + switch (settings.m_entropyCoding) { + case ALVR_CABAC: + AVUTIL.av_opt_set(encoder_ctx->priv_data, "coder", "ac", 0); + break; + case ALVR_CAVLC: + AVUTIL.av_opt_set(encoder_ctx->priv_data, "coder", "vlc", 0); + break; + } + break; case ALVR_CODEC_H265: encoder_ctx->profile = FF_PROFILE_HEVC_MAIN; diff --git a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp index 014ca36ede..b04e072559 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderNVENC.cpp @@ -208,6 +208,15 @@ void VideoEncoderNVENC::FillEncodeConfig(NV_ENC_INITIALIZE_PARAMS &initializePar config.intraRefreshCnt = Settings::Instance().m_nvencIntraRefreshCount; } + switch (Settings::Instance().m_entropyCoding) { + case ALVR_CABAC: + config.entropyCodingMode = NV_ENC_H264_ENTROPY_CODING_MODE_CABAC; + break; + case ALVR_CAVLC: + config.entropyCodingMode = NV_ENC_H264_ENTROPY_CODING_MODE_CAVLC; + break; + } + config.maxNumRefFrames = maxNumRefFrames; config.idrPeriod = gopLength; } diff --git a/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp b/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp index c209e59bec..75836cd4d2 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderSW.cpp @@ -59,7 +59,15 @@ void VideoEncoderSW::Initialize() { av_dict_set(&opt, "tune", "zerolatency", 0); switch (m_codec) { case ALVR_CODEC_H264: - m_codecContext->profile = Settings::Instance().m_use10bitEncoder ? FF_PROFILE_H264_HIGH_10 : FF_PROFILE_H264_HIGH; + m_codecContext->profile = Settings::Instance().m_use10bitEncoder ? FF_PROFILE_H264_HIGH_10_INTRA : (FF_PROFILE_H264_HIGH | FF_PROFILE_H264_INTRA); + switch (Settings::Instance().m_entropyCoding) { + case ALVR_CABAC: + av_dict_set(&opt, "coder", "ac", 0); + break; + case ALVR_CAVLC: + av_dict_set(&opt, "coder", "vlc", 0); + break; + } break; case ALVR_CODEC_H265: m_codecContext->profile = Settings::Instance().m_use10bitEncoder ? FF_PROFILE_HEVC_MAIN_10 : FF_PROFILE_HEVC_MAIN; diff --git a/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp b/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp index 8a50a359d2..cc43145f99 100644 --- a/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp +++ b/alvr/server/cpp/platform/win32/VideoEncoderVCE.cpp @@ -165,6 +165,16 @@ amf::AMFComponentPtr VideoEncoderVCE::MakeEncoder( amfEncoder->SetProperty(AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD, AMF_VIDEO_ENCODER_RATE_CONTROL_METHOD_LATENCY_CONSTRAINED_VBR); break; } + + switch (Settings::Instance().m_entropyCoding) { + case ALVR_CABAC: + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_CABAC_ENABLE, AMF_VIDEO_ENCODER_CABAC); + break; + case ALVR_CAVLC: + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_CABAC_ENABLE, AMF_VIDEO_ENCODER_CALV); + break; + } + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_TARGET_BITRATE, bitRateIn); amfEncoder->SetProperty(AMF_VIDEO_ENCODER_PEAK_BITRATE, bitRateIn); amfEncoder->SetProperty(AMF_VIDEO_ENCODER_FRAMESIZE, ::AMFConstructSize(width, height)); @@ -198,13 +208,13 @@ amf::AMFComponentPtr VideoEncoderVCE::MakeEncoder( amfEncoder->SetProperty(AMF_VIDEO_ENCODER_MAX_NUM_REFRAMES, 0); - amf::AMFCapsPtr caps; - if (amfEncoder->GetCaps(&caps) == AMF_OK) { - caps->GetProperty(AMF_VIDEO_ENCODER_CAPS_QUERY_TIMEOUT_SUPPORT, &m_hasQueryTimeout); - } - if (m_hasQueryTimeout) { - amfEncoder->SetProperty(AMF_VIDEO_ENCODER_QUERY_TIMEOUT, 1000); // 1s timeout - } + amf::AMFCapsPtr caps; + if (amfEncoder->GetCaps(&caps) == AMF_OK) { + caps->GetProperty(AMF_VIDEO_ENCODER_CAPS_QUERY_TIMEOUT_SUPPORT, &m_hasQueryTimeout); + } + if (m_hasQueryTimeout) { + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_QUERY_TIMEOUT, 1000); // 1s timeout + } } else { @@ -261,13 +271,13 @@ amf::AMFComponentPtr VideoEncoderVCE::MakeEncoder( amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_MAX_NUM_REFRAMES, 0); - amf::AMFCapsPtr caps; - if (amfEncoder->GetCaps(&caps) == AMF_OK) { - caps->GetProperty(AMF_VIDEO_ENCODER_CAPS_HEVC_QUERY_TIMEOUT_SUPPORT, &m_hasQueryTimeout); - } - if (m_hasQueryTimeout) { - amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_QUERY_TIMEOUT, 1000); // 1s timeout - } + amf::AMFCapsPtr caps; + if (amfEncoder->GetCaps(&caps) == AMF_OK) { + caps->GetProperty(AMF_VIDEO_ENCODER_CAPS_HEVC_QUERY_TIMEOUT_SUPPORT, &m_hasQueryTimeout); + } + if (m_hasQueryTimeout) { + amfEncoder->SetProperty(AMF_VIDEO_ENCODER_HEVC_QUERY_TIMEOUT, 1000); // 1s timeout + } } Debug("Configured %s.\n", pCodec); diff --git a/alvr/server/src/connection.rs b/alvr/server/src/connection.rs index 2f12a7bfb3..3db1603d16 100644 --- a/alvr/server/src/connection.rs +++ b/alvr/server/src/connection.rs @@ -251,6 +251,7 @@ async fn client_handshake( adapter_index: settings.video.adapter_index, codec: matches!(settings.video.codec, CodecType::HEVC) as _, rate_control_mode: settings.video.rate_control_mode as u32, + entropy_coding: settings.video.entropy_coding as u32, refresh_rate: fps as _, use_10bit_encoder: settings.video.use_10bit_encoder, use_preproc: amf_controls.use_preproc, diff --git a/alvr/session/src/lib.rs b/alvr/session/src/lib.rs index 23f5bb8697..9d61eebe61 100644 --- a/alvr/session/src/lib.rs +++ b/alvr/session/src/lib.rs @@ -67,6 +67,7 @@ pub struct OpenvrConfig { pub preproc_tor: u32, pub encoder_quality_preset: u32, pub rate_control_mode: u32, + pub entropy_coding: u32, pub force_sw_encoding: bool, pub sw_thread_count: u32, pub encode_bitrate_mbs: u64, diff --git a/alvr/session/src/settings.rs b/alvr/session/src/settings.rs index 1b12bb95ef..c0af488022 100644 --- a/alvr/session/src/settings.rs +++ b/alvr/session/src/settings.rs @@ -61,6 +61,14 @@ pub enum RateControlMode { VBR = 1, } +#[repr(u8)] +#[derive(SettingsSchema, Serialize, Deserialize, Clone)] +#[serde(tag = "type", content = "content")] +pub enum EntropyCoding { + CABAC = 0, + CAVLC = 1, +} + /// Except for preset, the value of these fields is not applied if == -1 (flag) #[derive(SettingsSchema, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -226,6 +234,9 @@ pub struct VideoDesc { #[schema(advanced)] pub rate_control_mode: RateControlMode, + #[schema(advanced)] + pub entropy_coding: EntropyCoding, + // #[schema(advanced)] // pub video_coding: VideoCoding, #[schema(advanced)] @@ -629,6 +640,9 @@ pub fn session_settings_default() -> SettingsDefault { variant: RateControlModeDefaultVariant::CBR, }, client_request_realtime_decoder: true, + entropy_coding: EntropyCodingDefault { + variant: EntropyCodingDefaultVariant::CAVLC, + }, use_10bit_encoder: false, sw_thread_count: 0, encode_bitrate_mbs: 30,