github/xrdp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add support for nvenc and accel_assist

Browse Source
This commit is contained in:
Jay Sorg
2024-05-23 02:26:40 -07:00
parent 6889a0731d
commit 6dcb8ffe79
22 changed files with 11017 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xrdp_accel_assist/xrdp_accel_assist_nvenc.c
+383
View File
@@ -0,0 +1,383 @@
/**
* xrdp: A Remote Desktop Protocol server.
*
* Copyright (C) Jay Sorg 2022-2024
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#if defined(HAVE_CONFIG_H)
#include <config_ac.h>
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "string_calls.h"
#include <epoxy/gl.h>
#include "encoder_headers/nvEncodeAPI_11_1.h"
#include "arch.h"
#include "os_calls.h"
#include "xrdp_accel_assist.h"
#include "xrdp_accel_assist_x11.h"
#include "xrdp_accel_assist_nvenc.h"
#include "log.h"
#define XH_NVENV_DEFAULT_QP 28
typedef NVENCSTATUS
(NVENCAPI *NvEncodeAPICreateInstanceProc)
(NV_ENCODE_API_FUNCTION_LIST *functionList);
static char g_lib_name[] = "libnvidia-encode.so";
static char g_lib_name1[] = "libnvidia-encode.so.1";
static char g_func_name[] = "NvEncodeAPICreateInstance";
static NvEncodeAPICreateInstanceProc g_NvEncodeAPICreateInstance = NULL;
static NV_ENCODE_API_FUNCTION_LIST g_enc_funcs;
static long g_lib = 0;
struct enc_info
{
int width;
int height;
int frameCount;
int pad0;
void *enc;
NV_ENC_OUTPUT_PTR bitstreamBuffer;
NV_ENC_INPUT_PTR mappedResource;
NV_ENC_BUFFER_FORMAT mappedBufferFmt;
NV_ENC_REGISTERED_PTR registeredResource;
};
/*****************************************************************************/
int
xrdp_accel_assist_nvenc_init(void)
{
NVENCSTATUS nv_error;
g_lib = g_load_library(g_lib_name);
if (g_lib == 0)
{
g_lib = g_load_library(g_lib_name1);
if (g_lib == 0)
{
LOG(LOG_LEVEL_ERROR, "load library for %s/%s failed", g_lib_name, g_lib_name1);
return 1;
}
}
g_NvEncodeAPICreateInstance = g_get_proc_address(g_lib, g_func_name);
if (g_NvEncodeAPICreateInstance == NULL)
{
LOG(LOG_LEVEL_ERROR, "get proc address for %s failed", g_func_name);
return 1;
}
g_memset(&g_enc_funcs, 0, sizeof(g_enc_funcs));
g_enc_funcs.version = NV_ENCODE_API_FUNCTION_LIST_VER;
nv_error = g_NvEncodeAPICreateInstance(&g_enc_funcs);
LOG(LOG_LEVEL_INFO, "NvEncodeAPICreateInstance rv %d", nv_error);
if (nv_error != NV_ENC_SUCCESS)
{
return 1;
}
return 0;
}
/*****************************************************************************/
int
xrdp_accel_assist_nvenc_create_encoder(int width, int height, int tex,
int tex_format, struct enc_info **ei)
{
NV_ENC_CREATE_BITSTREAM_BUFFER bitstreamParams;
NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS params;
NV_ENC_INITIALIZE_PARAMS createEncodeParams;
NV_ENC_MAP_INPUT_RESOURCE mapInputResource;
NV_ENC_INPUT_RESOURCE_OPENGL_TEX res;
NV_ENC_REGISTER_RESOURCE reg_res;
NV_ENC_CONFIG encCfg;
NVENCSTATUS nv_error;
struct enc_info *lei;
char *rateControlMode_str;
char *averageBitRate_str;
char *qp_str;
int qp_int;
int averageBitRate_int;
int rc_set;
lei = g_new0(struct enc_info, 1);
if (lei == NULL)
{
return 1;
}
g_memset(&params, 0, sizeof(params));
params.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER;
params.deviceType = NV_ENC_DEVICE_TYPE_OPENGL;
params.apiVersion = NVENCAPI_VERSION;
nv_error = g_enc_funcs.nvEncOpenEncodeSessionEx(&params, &(lei->enc));
LOG(LOG_LEVEL_INFO, "nvEncOpenEncodeSessionEx rv %d enc %p", nv_error, lei->enc);
if (nv_error != NV_ENC_SUCCESS)
{
g_free(lei);
return 1;
}
g_memset(&encCfg, 0, sizeof(encCfg));
encCfg.version = NV_ENC_CONFIG_VER;
encCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID;
encCfg.gopLength = NVENC_INFINITE_GOPLENGTH;
encCfg.frameIntervalP = 1; /* 1 + B_Frame_Count */
encCfg.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME;
encCfg.mvPrecision = NV_ENC_MV_PRECISION_QUARTER_PEL;
/* these env vars can be added / changed in sesman.ini SessionVariables
example
XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_CONSTQP
XRDP_NVENC_QP=30
or
XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_VBR
XRDP_NVENC_AVERAGE_BITRATE=2000000 */
rateControlMode_str = g_getenv("XRDP_NVENC_RATE_CONTROL_MODE");
averageBitRate_str = g_getenv("XRDP_NVENC_AVERAGE_BITRATE");
qp_str = g_getenv("XRDP_NVENC_QP");
rc_set = 0;
if (rateControlMode_str != NULL)
{
if (g_strcmp(rateControlMode_str, "NV_ENC_PARAMS_RC_CONSTQP") == 0)
{
if (qp_str != NULL)
{
qp_int = g_atoi(qp_str);
if ((qp_int >= 0) && (qp_int <= 51))
{
LOG(LOG_LEVEL_INFO,
"using NV_ENC_PARAMS_RC_CONSTQP qp %d",
qp_int);
encCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
encCfg.rcParams.constQP.qpInterP = qp_int;
encCfg.rcParams.constQP.qpInterB = qp_int;
encCfg.rcParams.constQP.qpIntra = qp_int;
rc_set = 1;
}
}
}
else if (g_strcmp(rateControlMode_str, "NV_ENC_PARAMS_RC_VBR") == 0)
{
if (averageBitRate_str != NULL)
{
averageBitRate_int = g_atoi(averageBitRate_str);
if ((averageBitRate_int >= 5000) &&
(averageBitRate_int <= 1000000000))
{
LOG(LOG_LEVEL_INFO,
"using NV_ENC_PARAMS_RC_VBR averageBitRate %d",
averageBitRate_int);
encCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR;
encCfg.rcParams.averageBitRate = averageBitRate_int;
rc_set = 1;
}
}
}
}
if (!rc_set)
{
LOG(LOG_LEVEL_INFO,
"using default NV_ENC_PARAMS_RC_CONSTQP qp %d",
XH_NVENV_DEFAULT_QP);
encCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP;
encCfg.rcParams.constQP.qpInterP = XH_NVENV_DEFAULT_QP;
encCfg.rcParams.constQP.qpInterB = XH_NVENV_DEFAULT_QP;
encCfg.rcParams.constQP.qpIntra = XH_NVENV_DEFAULT_QP;
rc_set = 1;
}
encCfg.encodeCodecConfig.h264Config.chromaFormatIDC = 1;
encCfg.encodeCodecConfig.h264Config.idrPeriod = NVENC_INFINITE_GOPLENGTH;
encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1;
encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0;
g_memset(&createEncodeParams, 0, sizeof(createEncodeParams));
createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER;
createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID;
createEncodeParams.encodeWidth = width;
createEncodeParams.encodeHeight = height;
createEncodeParams.darWidth = width;
createEncodeParams.darHeight = height;
createEncodeParams.frameRateNum = 30;
createEncodeParams.frameRateDen = 1;
createEncodeParams.enablePTD = 1;
createEncodeParams.encodeConfig = &encCfg;
nv_error = g_enc_funcs.nvEncInitializeEncoder(lei->enc,
&createEncodeParams);
LOG(LOG_LEVEL_INFO, "nvEncInitializeEncoder rv %d", nv_error);
if (nv_error != NV_ENC_SUCCESS)
{
g_free(lei);
return 1;
}
g_memset(&res, 0, sizeof(res));
res.texture = tex;
res.target = GL_TEXTURE_2D;
g_memset(&reg_res, 0, sizeof(reg_res));
reg_res.version = NV_ENC_REGISTER_RESOURCE_VER;
reg_res.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX;
reg_res.width = width;
reg_res.height = height;
if (tex_format == XH_YUV420)
{
reg_res.pitch = width;
reg_res.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12;
}
else
{
reg_res.pitch = width * 4;
reg_res.bufferFormat = NV_ENC_BUFFER_FORMAT_AYUV;
}
reg_res.resourceToRegister = &res;
reg_res.bufferUsage = NV_ENC_INPUT_IMAGE;
nv_error = g_enc_funcs.nvEncRegisterResource(lei->enc, &reg_res);
LOG(LOG_LEVEL_INFO, "nvEncRegisterResource rv %d", nv_error);
if (nv_error != NV_ENC_SUCCESS)
{
g_free(lei);
return 1;
}
g_memset(&mapInputResource, 0, sizeof(mapInputResource));
mapInputResource.version = NV_ENC_LOCK_INPUT_BUFFER_VER;
mapInputResource.registeredResource = reg_res.registeredResource;
nv_error = g_enc_funcs.nvEncMapInputResource(lei->enc, &mapInputResource);
LOG(LOG_LEVEL_INFO, "nvEncMapInputResource rv %d", nv_error);
if (nv_error != NV_ENC_SUCCESS)
{
g_free(lei);
return 1;
}
g_memset(&bitstreamParams, 0, sizeof(bitstreamParams));
bitstreamParams.version = NV_ENC_CREATE_BITSTREAM_BUFFER_VER;
nv_error = g_enc_funcs.nvEncCreateBitstreamBuffer(lei->enc,
&bitstreamParams);
LOG(LOG_LEVEL_INFO, "nvEncCreateBitstreamBuffer rv %d", nv_error);
if (nv_error != NV_ENC_SUCCESS)
{
g_free(lei);
return 1;
}
lei->bitstreamBuffer = bitstreamParams.bitstreamBuffer;
lei->mappedResource = mapInputResource.mappedResource;
lei->mappedBufferFmt = mapInputResource.mappedBufferFmt;
lei->registeredResource = reg_res.registeredResource;
lei->width = width;
lei->height = height;
*ei = lei;
return 0;
}
/*****************************************************************************/
int
xrdp_accel_assist_nvenc_delete_encoder(struct enc_info *ei)
{
g_enc_funcs.nvEncUnmapInputResource(ei->enc, ei->mappedResource);
g_enc_funcs.nvEncUnregisterResource(ei->enc, ei->registeredResource);
g_enc_funcs.nvEncDestroyBitstreamBuffer(ei->enc, ei->bitstreamBuffer);
g_enc_funcs.nvEncDestroyEncoder(ei->enc);
g_free(ei);
return 0;
}
/*****************************************************************************/
enum encoder_result
xrdp_accel_assist_nvenc_encode(struct enc_info *ei, int tex,
void *cdata, int *cdata_bytes,
int flags)
{
NV_ENC_PIC_PARAMS picParams;
NV_ENC_LOCK_BITSTREAM lockBitstream;
NVENCSTATUS nv_error;
enum encoder_result rv;
/* sync before encoding */
glFinish();
g_memset(&picParams, 0, sizeof(picParams));
picParams.version = NV_ENC_PIC_PARAMS_VER;
picParams.inputBuffer = ei->mappedResource;
picParams.bufferFmt = ei->mappedBufferFmt;
picParams.inputWidth = ei->width;
picParams.inputHeight = ei->height;
picParams.outputBitstream = ei->bitstreamBuffer;
picParams.inputTimeStamp = ei->frameCount;
picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME;
picParams.encodePicFlags = NV_ENC_PIC_FLAG_OUTPUT_SPSPPS;
if ((flags & XH_ENC_FLAGS_FORCEIDR) || (ei->frameCount < 1))
{
picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR;
LOG(LOG_LEVEL_INFO, "Forcing NVENC H264 IDR SPSPPS for frame id: %d",
ei->frameCount);
}
nv_error = g_enc_funcs.nvEncEncodePicture(ei->enc, &picParams);
rv = ENCODER_ERROR;
if (nv_error == NV_ENC_SUCCESS)
{
g_memset(&lockBitstream, 0, sizeof(lockBitstream));
lockBitstream.version = NV_ENC_LOCK_BITSTREAM_VER;
lockBitstream.outputBitstream = ei->bitstreamBuffer;
lockBitstream.doNotWait = 0;
nv_error = g_enc_funcs.nvEncLockBitstream(ei->enc, &lockBitstream);
if (nv_error == NV_ENC_SUCCESS)
{
if (*cdata_bytes >= ((int) (lockBitstream.bitstreamSizeInBytes)))
{
g_memcpy(cdata, lockBitstream.bitstreamBufferPtr,
lockBitstream.bitstreamSizeInBytes);
*cdata_bytes = lockBitstream.bitstreamSizeInBytes;
rv = INCREMENTAL_FRAME_ENCODED;
}
else
{
LOG(LOG_LEVEL_ERROR, "error not enough room %d %d",
*cdata_bytes,
(int) (lockBitstream.bitstreamSizeInBytes));
}
g_enc_funcs.nvEncUnlockBitstream(ei->enc,
lockBitstream.outputBitstream);
}
else
{
LOG(LOG_LEVEL_ERROR, "error nvEncLockBitstream %d",
nv_error);
}
ei->frameCount++;
}
else
{
LOG(LOG_LEVEL_ERROR, "error nvEncEncodePicture %d", nv_error);
}
if (rv == INCREMENTAL_FRAME_ENCODED
&& (picParams.encodePicFlags & NV_ENC_PIC_FLAG_FORCEIDR))
{
return KEY_FRAME_ENCODED;
}
return rv;
}