openh264
OpenH264 是一个开源的 H.264 编码和解码器,由思科系统开发并维护。它专为实时应用程序如 WebRTC 设计,提供了从基础到高级特性的广泛支持。OpenH264 的编码器支持从 Constrained Baseline Profile 到 5.2 级别,允许任意分辨率的编码,不限于 16x16 的倍数,并且具有自适应量化或恒定量化的速率控制、多切片选项、多线程自动用于多切片等特性。此外,它还支持高达 4 层的时序可伸缩性、单一输入的 4 种空间分辨率的 Spatial Simulcast、长期参考帧(LTR)、内存管理控制操作(MMCO)等功能。
帧间预测编码
视频帧间预测编码(Inter-frame prediction coding)是视频压缩技术中的一种关键方法,主要用于减少视频序列中时间维度上的冗余。这种编码方式依赖于视频帧之间的空间相关性,通过预测和补偿来减少数据量,从而实现高效的视频压缩。
帧间预测编码是视频编码中非常有效的技术,广泛应用于各种视频编码标准,如H.264/AVC、H.265/HEVC、VP9和AV1等。通过减少时间冗余,帧间预测编码显著提高了视频数据的压缩率,同时保持了视频质量。
帧间预测的核心技术主要包括以下几个方面:
运动估计(Motion Estimation, ME):
- 运动估计是指在参考帧中搜索与当前块最匹配的区域,以确定最佳匹配块的位置。
运动补偿(Motion Compensation, MC):
- 运动补偿使用运动估计得到的运动向量来预测当前块,通过补偿先前图像的抽样点来生成当前图像块的预测值。
宏块(Macroblock, MB)和子宏块(Sub-Macroblock)的树状结构分块:
- H.264 支持多种宏块分割方式,如16x16、16x8、8x16和8x8,以及更细致的子宏块分割,如8x8、8x4、4x8和4x4。
多参考帧预测:
- 特别是在B帧中,可以使用两个方向的参考帧(List0和List1)进行双向预测。
亚像素精度的运动估计:
- 除了整像素精度外,H.264还支持1/2像素和1/4像素的亚像素插值,以提高预测精度。
快速搜索算法:
- 为了减少计算复杂度,使用快速搜索算法来确定最佳匹配块。
运动向量的编码:
- 运动向量的编码通常涉及编码运动向量与预测值的差值(MVD),而不是直接编码运动向量本身。
预测模式的选择:
- H.264定义了多种预测模式,包括直接模式、双向模式、List0和List1模式等。
参考帧管理:
- 管理参考帧以确保解码器能够正确地重建预测图像。
变换/量化编码和熵编码:
- 这些步骤与帧内编码相同或相似,用于进一步压缩预测残差。
帧间预测通过这些技术有效地减少了视频序列中的时间冗余,提高了压缩效率。
openh264 帧间预测编码过程
- 帧间预测过程函数关系图:
- 关键模块说明:
- 通过API 函数
EncodeFrame
完成具体的编码过程,可执行程序封装该函数进行编码;- 通过二维数组g_pWelsSliceCoding[2][2]来完成具体的I 帧、P 帧,以及变化片数和非变化片数编码;
- 在非变化片数帧间编码过程中,在
WelsMdInterMbLoop
函数中循环处理每个宏块;- 在变化片数帧间编码过程中,在
WelsMdInterMbLoopOverDynamicSlice
函数中循环处理每个宏块;pfInterMd
函数指针指向具体的增强层WelsMdInterMbEnhancelayer
或基本层WelsMdInterMb
帧间预测过程;- 在基本层预测过程中,
WelsMdInterSecondaryModesEnc
函数完成二级帧间预测过程编码;- 在增强层预测过程中,
WelsMdSpatialelInterMbIlfmdNoilp
函数完成具体的预测过程编码;
- 帧间预测过程核心函数介绍:
WelsMdInterMbLoopOverDynamicSlice
函数
- 用于动态切片的宏块编码过程,循环处理每个 MB;
// Only for inter dynamic slicing
int32_t WelsMdInterMbLoopOverDynamicSlice (sWelsEncCtx* pEncCtx, SSlice* pSlice, void* pWelsMd,
const int32_t kiSliceFirstMbXY) {
SWelsMD* pMd = (SWelsMD*)pWelsMd;
SBitStringAux* pBs = pSlice->pSliceBsa;
SDqLayer* pCurLayer = pEncCtx->pCurDqLayer;
SSliceCtx* pSliceCtx = &pCurLayer->sSliceEncCtx;
SMbCache* pMbCache = &pSlice->sMbCacheInfo;
SMB* pMbList = pCurLayer->sMbDataP;
SMB* pCurMb = NULL;
int32_t iNumMbCoded = 0;
const int32_t kiTotalNumMb = pCurLayer->iMbWidth * pCurLayer->iMbHeight;
int32_t iNextMbIdx = kiSliceFirstMbXY;
int32_t iCurMbIdx = -1;
const int32_t kiMvdInterTableStride = pEncCtx->iMvdCostTableStride;
uint16_t* pMvdCostTable = &pEncCtx->pMvdCostTable[pEncCtx->iMvdCostTableSize];
const int32_t kiSliceIdx = pSlice->iSliceIdx;
const int32_t kiPartitionId = (kiSliceIdx % pEncCtx->iActiveThreadsNum);
const uint8_t kuiChromaQpIndexOffset = pCurLayer->sLayerInfo.pPpsP->uiChromaQpIndexOffset;
int32_t iEncReturn = ENC_RETURN_SUCCESS;
SDynamicSlicingStack sDss;
if (pEncCtx->pSvcParam->iEntropyCodingModeFlag) {
WelsInitSliceCabac (pEncCtx, pSlice);
sDss.iStartPos = sDss.iCurrentPos = 0;
sDss.pRestoreBuffer = pEncCtx->pDynamicBsBuffer[kiPartitionId];
} else {
sDss.iStartPos = BsGetBitsPos (pBs);
}
pSlice->iMbSkipRun = 0;
for (;;) {
//DYNAMIC_SLICING_ONE_THREAD - MultiD
//stack pBs pointer
pEncCtx->pFuncList->pfStashMBStatus (&sDss, pSlice, pSlice->iMbSkipRun);
//point to current pMb
iCurMbIdx = iNextMbIdx;
pCurMb = &pMbList[ iCurMbIdx ];
//step(1): set QP for the current MB
pEncCtx->pFuncList->pfRc.pfWelsRcMbInit (pEncCtx, pCurMb, pSlice);
// if already reaches the largest number of slices, set QPs to the upper bound
if (pSlice->bDynamicSlicingSliceSizeCtrlFlag) {
//a clearer logic may be:
//if there is no need from size control from the pSlice size, the QP will be decided by RC; else it will be set to the max QP
// however, there are some parameter updating in the rc_mb_init() function, so it cannot be skipped?
pCurMb->uiLumaQp = pEncCtx->pWelsSvcRc[pEncCtx->uiDependencyId].iMaxQp;
pCurMb->uiChromaQp = g_kuiChromaQpTable[CLIP3_QP_0_51 (pCurMb->uiLumaQp + kuiChromaQpIndexOffset)];
}
//step (2). save some vale for future use, initial pWelsMd
WelsMdIntraInit (pEncCtx, pCurMb, pMbCache, kiSliceFirstMbXY);
WelsMdInterInit (pEncCtx, pSlice, pCurMb, kiSliceFirstMbXY);
TRY_REENCODING:
WelsInitInterMDStruc (pCurMb, pMvdCostTable, kiMvdInterTableStride, pMd);
pEncCtx->pFuncList->pfInterMd (pEncCtx, pMd, pSlice, pCurMb, pMbCache);
//mb_qp
//step (4): save from the MD process from future use
WelsMdInterSaveSadAndRefMbType ((pCurLayer->pDecPic->uiRefMbType), pMbCache, pCurMb, pMd);
pEncCtx->pFuncList->pfMdBackgroundInfoUpdate (pCurLayer, pCurMb, pMbCache->bCollocatedPredFlag,
pEncCtx->pRefPic->iPictureType);
//step (5): update cache
UpdateNonZeroCountCache (pCurMb, pMbCache);
//step (6): begin to write bit stream; if the pSlice size is controlled, the writing may be skipped
iEncReturn = pEncCtx->pFuncList->pfWelsSpatialWriteMbSyn (pEncCtx, pSlice, pCurMb);
if (iEncReturn == ENC_RETURN_VLCOVERFLOWFOUND && (pCurMb->uiLumaQp < 50)) {
pSlice->iMbSkipRun = pEncCtx->pFuncList->pfStashPopMBStatus (&sDss, pSlice);
UpdateQpForOverflow (pCurMb, kuiChromaQpIndexOffset);
goto TRY_REENCODING;
}
if (ENC_RETURN_SUCCESS != iEncReturn)
return iEncReturn;
//DYNAMIC_SLICING_ONE_THREAD - MultiD
sDss.iCurrentPos = pEncCtx->pFuncList->pfGetBsPosition (pSlice);
if (DynSlcJudgeSliceBoundaryStepBack (pEncCtx, pSlice, pSliceCtx, pCurMb, &sDss)) {
pSlice->iMbSkipRun = pEncCtx->pFuncList->pfStashPopMBStatus (&sDss, pSlice);
pCurLayer->LastCodedMbIdxOfPartition[kiPartitionId] = iCurMbIdx -
1; // update LastCodedMbIdxOfPartition, need to -1 due to stepping back
++ pCurLayer->NumSliceCodedOfPartition[kiPartitionId];
break;
}
//step (7): reconstruct current MB
pCurMb->uiSliceIdc = kiSliceIdx;
OutputPMbWithoutConstructCsRsNoCopy (pEncCtx, pCurLayer, pSlice, pCurMb);
#if defined(MB_TYPES_CHECK)
WelsCountMbType (pEncCtx->sPerInfo.iMbCount, P_SLICE, pCurMb);
#endif//MB_TYPES_CHECK
//step (8): update status and other parameters
pEncCtx->pFuncList->pfRc.pfWelsRcMbInfoUpdate (pEncCtx, pCurMb, pMd->iCostLuma, pSlice);
/*judge if all pMb in cur pSlice has been encoded*/
++ iNumMbCoded;
iNextMbIdx = WelsGetNextMbOfSlice (pCurLayer, iCurMbIdx);
//whether all of MB in current pSlice encoded or not
if (iNextMbIdx == -1 || iNextMbIdx >= kiTotalNumMb || iNumMbCoded >= kiTotalNumMb) {
pCurLayer->LastCodedMbIdxOfPartition[kiPartitionId] = iCurMbIdx;
++ pCurLayer->NumSliceCodedOfPartition[kiPartitionId];
break;
}
}
if (pSlice->iMbSkipRun) {
BsWriteUE (pBs, pSlice->iMbSkipRun);
}
return iEncReturn;
}
}//namespace WelsEnc
WelsMdInterMbLoop
函数
- 用于固定切片的宏块编码过程,循环处理每个 MB;
// for inter non-dynamic pSlice
int32_t WelsMdInterMbLoop (sWelsEncCtx* pEncCtx, SSlice* pSlice, void* pWelsMd, const int32_t kiSliceFirstMbXY) {
SWelsMD* pMd = (SWelsMD*)pWelsMd;
SBitStringAux* pBs = pSlice->pSliceBsa;
SDqLayer* pCurLayer = pEncCtx->pCurDqLayer;
SMbCache* pMbCache = &pSlice->sMbCacheInfo;
SMB* pMbList = pCurLayer->sMbDataP;
SMB* pCurMb = NULL;
int32_t iNumMbCoded = 0;
int32_t iNextMbIdx = kiSliceFirstMbXY;
int32_t iCurMbIdx = -1;
const int32_t kiTotalNumMb = pCurLayer->iMbWidth * pCurLayer->iMbHeight;
const int32_t kiMvdInterTableStride = pEncCtx->iMvdCostTableStride;
uint16_t* pMvdCostTable = &pEncCtx->pMvdCostTable[pEncCtx->iMvdCostTableSize];
const int32_t kiSliceIdx = pSlice->iSliceIdx;
const uint8_t kuiChromaQpIndexOffset = pCurLayer->sLayerInfo.pPpsP->uiChromaQpIndexOffset;
int32_t iEncReturn = ENC_RETURN_SUCCESS;
SDynamicSlicingStack sDss;
if (pEncCtx->pSvcParam->iEntropyCodingModeFlag) {
WelsInitSliceCabac (pEncCtx, pSlice);
sDss.pRestoreBuffer = NULL;
sDss.iStartPos = sDss.iCurrentPos = 0;
}
pSlice->iMbSkipRun = 0;
for (;;) {
if (!pEncCtx->pSvcParam->iEntropyCodingModeFlag)
pEncCtx->pFuncList->pfStashMBStatus (&sDss, pSlice, pSlice->iMbSkipRun);
//point to current pMb
iCurMbIdx = iNextMbIdx;
pCurMb = &pMbList[ iCurMbIdx ];
//step(1): set QP for the current MB
pEncCtx->pFuncList->pfRc.pfWelsRcMbInit (pEncCtx, pCurMb, pSlice);
//step (2). save some vale for future use, initial pWelsMd
WelsMdIntraInit (pEncCtx, pCurMb, pMbCache, kiSliceFirstMbXY);
WelsMdInterInit (pEncCtx, pSlice, pCurMb, kiSliceFirstMbXY);
TRY_REENCODING:
WelsInitInterMDStruc (pCurMb, pMvdCostTable, kiMvdInterTableStride, pMd);
pEncCtx->pFuncList->pfInterMd (pEncCtx, pMd, pSlice, pCurMb, pMbCache);
//mb_qp
//step (4): save from the MD process from future use
WelsMdInterSaveSadAndRefMbType ((pCurLayer->pDecPic->uiRefMbType), pMbCache, pCurMb, pMd);
pEncCtx->pFuncList->pfMdBackgroundInfoUpdate (pCurLayer, pCurMb, pMbCache->bCollocatedPredFlag,
pEncCtx->pRefPic->iPictureType);
//step (5): update cache
UpdateNonZeroCountCache (pCurMb, pMbCache);
//step (6): begin to write bit stream; if the pSlice size is controlled, the writing may be skipped
iEncReturn = pEncCtx->pFuncList->pfWelsSpatialWriteMbSyn (pEncCtx, pSlice, pCurMb);
if (!pEncCtx->pSvcParam->iEntropyCodingModeFlag) {
if (iEncReturn == ENC_RETURN_VLCOVERFLOWFOUND && (pCurMb->uiLumaQp < 50)) {
pSlice->iMbSkipRun = pEncCtx->pFuncList->pfStashPopMBStatus (&sDss, pSlice);
UpdateQpForOverflow (pCurMb, kuiChromaQpIndexOffset);
goto TRY_REENCODING;
}
}
if (ENC_RETURN_SUCCESS != iEncReturn)
return iEncReturn;
//step (7): reconstruct current MB
pCurMb->uiSliceIdc = kiSliceIdx;
OutputPMbWithoutConstructCsRsNoCopy (pEncCtx, pCurLayer, pSlice, pCurMb);
#if defined(MB_TYPES_CHECK)
WelsCountMbType (pEncCtx->sPerInfo.iMbCount, P_SLICE, pCurMb);
#endif//MB_TYPES_CHECK
//step (8): update status and other parameters
pEncCtx->pFuncList->pfRc.pfWelsRcMbInfoUpdate (pEncCtx, pCurMb, pMd->iCostLuma, pSlice);
/*judge if all pMb in cur pSlice has been encoded*/
++ iNumMbCoded;
iNextMbIdx = WelsGetNextMbOfSlice (pCurLayer, iCurMbIdx);
//whether all of MB in current pSlice encoded or not
if (iNextMbIdx == -1 || iNextMbIdx >= kiTotalNumMb || iNumMbCoded >= kiTotalNumMb) {
break;
}
}
if (pSlice->iMbSkipRun) {
BsWriteUE (pBs, pSlice->iMbSkipRun);
}
return iEncReturn;
}
WelsMdInterMb
函数
- 基本层的预测编码的核心实现函数,主要用模式决策等过程;
void WelsMdInterMb (sWelsEncCtx* pEncCtx, SWelsMD* pWelsMd, SSlice* pSlice, SMB* pCurMb, SMbCache* pUnused) {
SDqLayer* pCurDqLayer = pEncCtx->pCurDqLayer;
SMbCache* pMbCache = &pSlice->sMbCacheInfo;
const uint32_t kuiNeighborAvail = pCurMb->uiNeighborAvail;
const int32_t kiMbWidth = pCurDqLayer->iMbWidth;
const SMB* top_mb = pCurMb - kiMbWidth;
const bool bMbLeftAvailPskip = ((kuiNeighborAvail & LEFT_MB_POS) ? IS_SKIP ((pCurMb - 1)->uiMbType) : false);
const bool bMbTopAvailPskip = ((kuiNeighborAvail & TOP_MB_POS) ? IS_SKIP (top_mb->uiMbType) : false);
const bool bMbTopLeftAvailPskip = ((kuiNeighborAvail & TOPLEFT_MB_POS) ? IS_SKIP ((top_mb - 1)->uiMbType) : false);
const bool bMbTopRightAvailPskip = ((kuiNeighborAvail & TOPRIGHT_MB_POS) ? IS_SKIP ((top_mb + 1)->uiMbType) : false);
bool bTrySkip = bMbLeftAvailPskip || bMbTopAvailPskip || bMbTopLeftAvailPskip || bMbTopRightAvailPskip;
bool bKeepSkip = bMbLeftAvailPskip && bMbTopAvailPskip && bMbTopRightAvailPskip;
bool bSkip = false;
//try BGD skip
if (pEncCtx->pFuncList->pfInterMdBackgroundDecision (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, &bKeepSkip)) {
return;
}
//try static or scrolled Pskip
if (pEncCtx->pFuncList->pfSCDPSkipDecision (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache)) {
return;
}
//step 1: try SKIP
bSkip = WelsMdInterJudgePskip (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bTrySkip);
if (bSkip) {
if (bKeepSkip) {
WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);
return;
}
} else {
PredictSad (pMbCache->sMvComponents.iRefIndexCache, pMbCache->iSadCost, 0, &pWelsMd->iSadPredMb);
//step 2: P_16x16
pWelsMd->iCostLuma = WelsMdP16x16 (pEncCtx->pFuncList, pCurDqLayer, pWelsMd, pSlice, pCurMb);
pCurMb->uiMbType = MB_TYPE_16x16;
}
WelsMdInterSecondaryModesEnc (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bSkip);
}
WelsMdInterSecondaryModesEnc
函数
- 基本层帧间二级模式的编码实现函数,主要用与判断出了 skip 和 p16x16 块类型之外的模式决策;
void WelsMdInterSecondaryModesEnc (sWelsEncCtx* pEncCtx, SWelsMD* pWelsMd, SSlice* pSlice, SMB* pCurMb,
SMbCache* pMbCache, const bool bSkip) {
//step 2: Intra
const bool kbTrySkip = pEncCtx->pFuncList->pfFirstIntraMode (pEncCtx, pWelsMd, pCurMb, pMbCache);
if (kbTrySkip)
return;
if (bSkip) {
WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);
} else {
//Step 3: SubP16 MD
pEncCtx->pFuncList->pfSetScrollingMv (pEncCtx->pVaa, pWelsMd); //SCC
pEncCtx->pFuncList->pfInterFineMd (pEncCtx, pWelsMd, pSlice, pCurMb, pWelsMd->iCostLuma);
//refinement for inter type
WelsMdInterMbRefinement (pEncCtx, pWelsMd, pCurMb, pMbCache);
//step 7: invoke encoding
WelsMdInterEncode (pEncCtx, pSlice, pCurMb, pMbCache);
//step 8: double check Pskip
WelsMdInterDoubleCheckPskip (pCurMb, pMbCache);
}
}
WelsMdSpatialelInterMbIlfmdNoilp
函数
- 增强层的预测编码的核心实现函数,主要用模式决策等过程;
//
// MD for enhancement layers
//
void WelsMdSpatialelInterMbIlfmdNoilp (sWelsEncCtx* pEncCtx, SWelsMD* pWelsMd, SSlice* pSlice,
SMB* pCurMb, const Mb_Type kuiRefMbType) {
SDqLayer* pCurDqLayer = pEncCtx->pCurDqLayer;
SMbCache* pMbCache = &pSlice->sMbCacheInfo;
const uint32_t kuiNeighborAvail = pCurMb->uiNeighborAvail;
const int32_t kiMbWidth = pCurDqLayer->iMbWidth;
const SMB* kpTopMb = pCurMb - kiMbWidth;
const bool kbMbLeftAvailPskip = ((kuiNeighborAvail & LEFT_MB_POS) ? IS_SKIP ((pCurMb - 1)->uiMbType) : false);
const bool kbMbTopAvailPskip = ((kuiNeighborAvail & TOP_MB_POS) ? IS_SKIP (kpTopMb->uiMbType) : false);
const bool kbMbTopLeftAvailPskip = ((kuiNeighborAvail & TOPLEFT_MB_POS) ? IS_SKIP ((kpTopMb - 1)->uiMbType) : false);
const bool kbMbTopRightAvailPskip = ((kuiNeighborAvail & TOPRIGHT_MB_POS) ? IS_SKIP ((kpTopMb + 1)->uiMbType) : false);
bool bTrySkip = kbMbLeftAvailPskip | kbMbTopAvailPskip | kbMbTopLeftAvailPskip | kbMbTopRightAvailPskip;
bool bKeepSkip = kbMbLeftAvailPskip & kbMbTopAvailPskip & kbMbTopRightAvailPskip;
bool bSkip = false;
if (pEncCtx->pFuncList->pfInterMdBackgroundDecision (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, &bKeepSkip)) {
return;
}
//step 1: try SKIP
bSkip = WelsMdInterJudgePskip (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bTrySkip);
if (bSkip && bKeepSkip) {
WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);
return;
}
if (! IS_SVC_INTRA (kuiRefMbType)) {
if (!bSkip) {
PredictSad (pMbCache->sMvComponents.iRefIndexCache, pMbCache->iSadCost, 0, &pWelsMd->iSadPredMb);
//step 2: P_16x16
pWelsMd->iCostLuma = WelsMdP16x16 (pEncCtx->pFuncList, pCurDqLayer, pWelsMd, pSlice, pCurMb);
pCurMb->uiMbType = MB_TYPE_16x16;
}
WelsMdInterSecondaryModesEnc (pEncCtx, pWelsMd, pSlice, pCurMb, pMbCache, bSkip);
} else { //BLMODE == SVC_INTRA
//initial prediction memory for I_16x16
const int32_t kiCostI16x16 = WelsMdI16x16 (pEncCtx->pFuncList, pEncCtx->pCurDqLayer, pMbCache, pWelsMd->iLambda);
if (bSkip && (pWelsMd->iCostLuma <= kiCostI16x16)) {
WelsMdInterDecidedPskip (pEncCtx, pSlice, pCurMb, pMbCache);
} else {
pWelsMd->iCostLuma = kiCostI16x16;
pCurMb->uiMbType = MB_TYPE_INTRA16x16;
WelsMdIntraSecondaryModesEnc (pEncCtx, pWelsMd, pCurMb, pMbCache);
}
}
}