diff --git a/3rdparty/h264/h264.h b/3rdparty/h264/h264.h index 295406ed3..3df7a9c44 100644 --- a/3rdparty/h264/h264.h +++ b/3rdparty/h264/h264.h @@ -282,6 +282,41 @@ namespace h264 { }; + enum PROFILE_IDC + { + PROFILE_IDC_BASELINE = 66, + PROFILE_IDC_MAIN = 77, + PROFILE_IDC_EXTENDED = 88, + PROFILE_IDC_HIGH = 100, + PROFILE_IDC_HIGH_10 = 110, + PROFILE_IDC_HIGH_422 = 122, + PROFILE_IDC_HIGH_444 = 244, + }; + + enum LIMIT + { + MAX_SPS_ID = 31, + MAX_PPS_ID = 255, + MAX_CHROMA_FORMAT_IDC = 3, + MAX_BIT_DEPTH_MINUS8 = 6, + MAX_LOG2_MINUS4 = 12, + MAX_PIC_ORDER_CNT_TYPE = 2, + MAX_NUM_REF_FRAMES_IN_PIC_ORDER_CNT_CYCLE = 255, + MAX_NUM_REF_FRAMES = 16, + MAX_NUM_REF_IDX_ACTIVE_MINUS1 = 31, + MAX_WEIGHTED_BIPRED_IDC = 2, + PIC_INIT_QP = 26, + MAX_QP = 51, + QP_BD_OFFSET_PER_BIT = 6, + MAX_CHROMA_QP_INDEX_OFFSET = 12, + MAX_SLICE_TYPE = 9, + MAX_CABAC_INIT_IDC = 2, + MAX_DISABLE_DEBLOCKING_FILTER_IDC = 2, + MAX_DEBLOCKING_OFFSET_DIV2 = 6, + MAX_IDR_PIC_ID = 65535, + MAX_REDUNDANT_PIC_CNT = 127, + }; + enum NAL_REF_IDC { NAL_REF_IDC_PRIORITY_HIGHEST = 3, @@ -635,7 +670,7 @@ namespace h264 { sps->offset_for_non_ref_pic = b->se(); sps->offset_for_top_to_bottom_field = b->se(); sps->num_ref_frames_in_pic_order_cnt_cycle = b->ue(); - for (int i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++) + for (int i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle && i < 256; i++) { sps->offset_for_ref_frame[i] = b->se(); } @@ -678,14 +713,14 @@ namespace h264 { pps->slice_group_map_type = b->ue(); if (pps->slice_group_map_type == 0) { - for (int i_group = 0; i_group <= pps->num_slice_groups_minus1; i_group++) + for (int i_group = 0; i_group <= pps->num_slice_groups_minus1 && i_group < 8; i_group++) { pps->run_length_minus1[i_group] = b->ue(); } } else if (pps->slice_group_map_type == 2) { - for (int i_group = 0; i_group < pps->num_slice_groups_minus1; i_group++) + for (int i_group = 0; i_group < pps->num_slice_groups_minus1 && i_group < 8; i_group++) { pps->top_left[i_group] = b->ue(); pps->bottom_right[i_group] = b->ue(); @@ -701,7 +736,7 @@ namespace h264 { else if (pps->slice_group_map_type == 6) { pps->pic_size_in_map_units_minus1 = b->ue(); - for (int i = 0; i <= pps->pic_size_in_map_units_minus1; i++) + for (int i = 0; i <= pps->pic_size_in_map_units_minus1 && i < 256; i++) { int v = intlog2(pps->num_slice_groups_minus1 + 1); pps->slice_group_id[i] = b->u(v); @@ -786,7 +821,7 @@ namespace h264 { { sh->rplr.reorder_l0.long_term_pic_num[n] = b->ue(); } - } while (sh->rplr.reorder_l0.reordering_of_pic_nums_idc[n] != 3 && !b->eof()); + } while (sh->rplr.reorder_l0.reordering_of_pic_nums_idc[n] != 3 && !b->eof() && n < 63); } } if (is_slice_type(sh->slice_type, SH_SLICE_TYPE_B)) @@ -808,7 +843,7 @@ namespace h264 { { sh->rplr.reorder_l1.long_term_pic_num[n] = b->ue(); } - } while (sh->rplr.reorder_l1.reordering_of_pic_nums_idc[n] != 3 && !b->eof()); + } while (sh->rplr.reorder_l1.reordering_of_pic_nums_idc[n] != 3 && !b->eof() && n < 63); } } } @@ -820,7 +855,7 @@ namespace h264 { { sh->pwt.chroma_log2_weight_denom = b->ue(); } - for (int i = 0; i <= sh->num_ref_idx_l0_active_minus1; i++) + for (int i = 0; i <= sh->num_ref_idx_l0_active_minus1 && i < 64; i++) { sh->pwt.luma_weight_l0_flag[i] = b->u1(); if (sh->pwt.luma_weight_l0_flag[i]) @@ -843,7 +878,7 @@ namespace h264 { } if (is_slice_type(sh->slice_type, SH_SLICE_TYPE_B)) { - for (int i = 0; i <= sh->num_ref_idx_l1_active_minus1; i++) + for (int i = 0; i <= sh->num_ref_idx_l1_active_minus1 && i < 64; i++) { sh->pwt.luma_weight_l1_flag[i] = b->u1(); if (sh->pwt.luma_weight_l1_flag[i]) @@ -901,7 +936,7 @@ namespace h264 { { sh->drpm.max_long_term_frame_idx_plus1[n] = b->ue(); } - } while (sh->drpm.memory_management_control_operation[n] != 0 && !b->eof()); + } while (sh->drpm.memory_management_control_operation[n] != 0 && !b->eof() && n < 63); } } } diff --git a/src/bgfx.cpp b/src/bgfx.cpp index 975aad434..03fa04d55 100644 --- a/src/bgfx.cpp +++ b/src/bgfx.cpp @@ -13,6 +13,10 @@ #include "topology.h" +#if BGFX_CONFIG_VIDEO +# include "video.h" +#endif // BGFX_CONFIG_VIDEO + #if BX_PLATFORM_OSX || BX_PLATFORM_IOS || BX_PLATFORM_VISIONOS # include #elif BX_PLATFORM_WINDOWS @@ -6524,6 +6528,55 @@ namespace bgfx if (kVideoDecoderInitMagic == init.magic && init.codec < VideoCodec::Count) { + if (0 == (g_caps.supported & BGFX_CAPS_VIDEO_DECODE) + || 0 == g_caps.codecs[init.codec]) + { + BX_WARN(false, "Video codec %d isn't supported by the renderer; video decoder not created.", init.codec); + release(_mem); + + return BGFX_INVALID_HANDLE; + } + +#if BGFX_CONFIG_VIDEO + if (VideoCodec::H264 == init.codec) + { + h264::SPS sps = {}; + + bool valid = true + && NULL != init.parameterSets + && peekH264Sps(init.parameterSets, init.parameterSetsSize, sps) + && isSpsInRange(sps) + ; + + if (valid) + { + const uint8_t chroma = 1 == sps.chroma_format_idc ? 0 + : 2 == sps.chroma_format_idc ? 2 + : 3 == sps.chroma_format_idc ? 4 + : 1 + ; + const uint8_t bitDepth = uint8_t(8 + bx::max(sps.bit_depth_luma_minus8, sps.bit_depth_chroma_minus8) ); + const uint32_t codedWidth = uint32_t(sps.pic_width_in_mbs_minus1 + 1) * 16; + const uint32_t codedHeight = uint32_t(sps.pic_height_in_map_units_minus1 + 1) * 16 * (0 != sps.frame_mbs_only_flag ? 1 : 2); + const uint8_t numRefs = uint8_t(bx::clamp(sps.num_ref_frames, 0, 16) ); + + valid = true + && codedWidth <= UINT16_MAX + && codedHeight <= UINT16_MAX + && isVideoCodecValid(init.codec, chroma, bitDepth, uint16_t(codedWidth), uint16_t(codedHeight), numRefs + 1, numRefs) + ; + } + + if (!valid) + { + BX_WARN(false, "H.264 parameter sets describe a stream the renderer doesn't decode; video decoder not created."); + release(_mem); + + return BGFX_INVALID_HANDLE; + } + } +#endif // BGFX_CONFIG_VIDEO + flags |= BGFX_TEXTURE_INTERNAL_VIDEO_DECODE_DST; } } diff --git a/src/bgfx_p.h b/src/bgfx_p.h index 16dcd6a4c..1a1b6882c 100644 --- a/src/bgfx_p.h +++ b/src/bgfx_p.h @@ -7935,6 +7935,33 @@ namespace bgfx return; } + + const VideoDecoderFrame* frame = (const VideoDecoderFrame*)_mem->data; + + if (kVideoDecoderFrameMagic != frame->magic) + { + BX_WARN(false + , "Video decode update has invalid magic 0x%08x (expected 0x%08x); skipping." + , frame->magic + , kVideoDecoderFrameMagic + ); + release(_mem); + + return; + } + + if (0 != frame->numAus + && (NULL == frame->bitstream || NULL == frame->aus) ) + { + BX_WARN(false + , "Video decode update has %d access units but no %s; skipping." + , frame->numAus + , NULL == frame->bitstream ? "bitstream" : "access unit array" + ); + release(_mem); + + return; + } } else { diff --git a/src/video.h b/src/video.h index bf2de9b98..0680c4e1b 100644 --- a/src/video.h +++ b/src/video.h @@ -18,6 +18,47 @@ namespace bgfx ; } + struct VideoPictureRect + { + uint32_t x; + uint32_t y; + uint32_t width; + uint32_t height; + }; + + inline VideoPictureRect getPictureRect(const h264::SPS& _sps) + { + const uint32_t frameMbsOnly = 0 != _sps.frame_mbs_only_flag ? 1 : 0; + const uint32_t codedWidth = uint32_t(_sps.pic_width_in_mbs_minus1 + 1) * 16; + const uint32_t codedHeight = uint32_t(_sps.pic_height_in_map_units_minus1 + 1) * 16 * (2 - frameMbsOnly); + + if (0 == _sps.frame_cropping_flag) + { + return { .x = 0, .y = 0, .width = codedWidth, .height = codedHeight }; + } + + const uint32_t cropUnitX = 2; + const uint32_t cropUnitY = 2 * (2 - frameMbsOnly); + const uint32_t left = uint32_t(_sps.frame_crop_left_offset) * cropUnitX; + const uint32_t right = uint32_t(_sps.frame_crop_right_offset) * cropUnitX; + const uint32_t top = uint32_t(_sps.frame_crop_top_offset) * cropUnitY; + const uint32_t bottom = uint32_t(_sps.frame_crop_bottom_offset) * cropUnitY; + + if (left + right >= codedWidth + || top + bottom >= codedHeight) + { + return { .x = 0, .y = 0, .width = codedWidth, .height = codedHeight }; + } + + return + { + .x = left, + .y = top, + .width = codedWidth - left - right, + .height = codedHeight - top - bottom, + }; + } + inline void getYuvToRgb(float _result[12], uint8_t _matrixCoefficients, bool _fullRange) { float kr = 0.2126f; @@ -217,6 +258,116 @@ namespace bgfx return poc; } + inline bool isInRange(int32_t _value, int32_t _min, int32_t _max) + { + return _min <= _value && _value <= _max; + } + + static constexpr int32_t kVideoMaxH264SizeInMbsMinus1 = 1023; + + inline bool isDecodableProfile(int32_t _profileIdc) + { + return false + || h264::PROFILE_IDC_BASELINE == _profileIdc + || h264::PROFILE_IDC_MAIN == _profileIdc + || h264::PROFILE_IDC_HIGH == _profileIdc + ; + } + + inline bool isSpsInRange(const h264::SPS& _sps) + { + return true + && isDecodableProfile(_sps.profile_idc) + && isInRange(_sps.seq_parameter_set_id, 0, h264::MAX_SPS_ID) + && isInRange(_sps.chroma_format_idc, 0, h264::MAX_CHROMA_FORMAT_IDC) + && isInRange(_sps.bit_depth_luma_minus8, 0, h264::MAX_BIT_DEPTH_MINUS8) + && isInRange(_sps.bit_depth_chroma_minus8, 0, h264::MAX_BIT_DEPTH_MINUS8) + && isInRange(_sps.log2_max_frame_num_minus4, 0, h264::MAX_LOG2_MINUS4) + && isInRange(_sps.pic_order_cnt_type, 0, h264::MAX_PIC_ORDER_CNT_TYPE) + && isInRange(_sps.log2_max_pic_order_cnt_lsb_minus4, 0, h264::MAX_LOG2_MINUS4) + && isInRange(_sps.num_ref_frames_in_pic_order_cnt_cycle, 0, h264::MAX_NUM_REF_FRAMES_IN_PIC_ORDER_CNT_CYCLE) + && isInRange(_sps.num_ref_frames, 0, h264::MAX_NUM_REF_FRAMES) + && isInRange(_sps.pic_width_in_mbs_minus1, 0, kVideoMaxH264SizeInMbsMinus1) + && isInRange(_sps.pic_height_in_map_units_minus1, 0, kVideoMaxH264SizeInMbsMinus1) + ; + } + + inline bool isPpsInRange(const h264::PPS& _pps, const h264::SPS& _sps) + { + const int32_t qpBdOffset = h264::QP_BD_OFFSET_PER_BIT*_sps.bit_depth_luma_minus8; + const int32_t maxQpInit = h264::MAX_QP - h264::PIC_INIT_QP; + + return true + && isInRange(_pps.pic_parameter_set_id, 0, h264::MAX_PPS_ID) + && isInRange(_pps.seq_parameter_set_id, 0, h264::MAX_SPS_ID) + && 0 == _pps.num_slice_groups_minus1 + && isInRange(_pps.num_ref_idx_l0_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && isInRange(_pps.num_ref_idx_l1_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && isInRange(_pps.weighted_bipred_idc, 0, h264::MAX_WEIGHTED_BIPRED_IDC) + && isInRange(_pps.pic_init_qp_minus26, -h264::PIC_INIT_QP - qpBdOffset, maxQpInit) + && isInRange(_pps.pic_init_qs_minus26, -h264::PIC_INIT_QP, maxQpInit) + && isInRange(_pps.chroma_qp_index_offset, -h264::MAX_CHROMA_QP_INDEX_OFFSET, h264::MAX_CHROMA_QP_INDEX_OFFSET) + && isInRange(_pps.second_chroma_qp_index_offset, -h264::MAX_CHROMA_QP_INDEX_OFFSET, h264::MAX_CHROMA_QP_INDEX_OFFSET) + ; + } + + inline bool isSliceHeaderInRange(const h264::SliceHeader& _sh, const h264::PPS& _pps, const h264::SPS& _sps) + { + const int32_t qpBdOffset = h264::QP_BD_OFFSET_PER_BIT*_sps.bit_depth_luma_minus8; + const int32_t sliceQp = h264::PIC_INIT_QP + _pps.pic_init_qp_minus26 + _sh.slice_qp_delta; + const int32_t frameHeight = (_sps.pic_height_in_map_units_minus1 + 1) * (0 != _sps.frame_mbs_only_flag ? 1 : 2); + const int32_t picSizeInMbs = (_sps.pic_width_in_mbs_minus1 + 1) * frameHeight; + const int32_t mbaff = 0 != _sps.mb_adaptive_frame_field_flag && 0 == _sh.field_pic_flag ? 1 : 0; + + return true + && isInRange(_sh.first_mb_in_slice * (1 + mbaff), 0, picSizeInMbs - 1) + && isInRange(_sh.slice_type, 0, h264::MAX_SLICE_TYPE) + && isInRange(sliceQp, -qpBdOffset, h264::MAX_QP) + && isInRange(_sh.num_ref_idx_l0_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && isInRange(_sh.num_ref_idx_l1_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && isInRange(_sh.cabac_init_idc, 0, h264::MAX_CABAC_INIT_IDC) + && isInRange(_sh.disable_deblocking_filter_idc, 0, h264::MAX_DISABLE_DEBLOCKING_FILTER_IDC) + && isInRange(_sh.slice_alpha_c0_offset_div2, -h264::MAX_DEBLOCKING_OFFSET_DIV2, h264::MAX_DEBLOCKING_OFFSET_DIV2) + && isInRange(_sh.slice_beta_offset_div2, -h264::MAX_DEBLOCKING_OFFSET_DIV2, h264::MAX_DEBLOCKING_OFFSET_DIV2) + && isInRange(_sh.idr_pic_id, 0, h264::MAX_IDR_PIC_ID) + && isInRange(_sh.redundant_pic_cnt, 0, h264::MAX_REDUNDANT_PIC_CNT) + ; + } + + inline bool peekH264Sps(const uint8_t* _data, uint32_t _size, h264::SPS& _outSps) + { + h264::Bitstream bs; + bs.init(_data, _size); + + while (h264::find_next_nal(&bs) ) + { + const uint8_t* nalStart = bs.p; + const uint32_t nalSize = nalUnitSize(nalStart, uint32_t(bs.end - bs.p) ); + + uint8_t rbsp[1024]; + const uint32_t rbspSize = stripEmulationPrevention(rbsp, nalStart, bx::min(nalSize, sizeof(rbsp) ) ); + + h264::Bitstream nalBs; + nalBs.init(rbsp, rbspSize); + + h264::NALHeader nal = {}; + + if (h264::read_nal_header(&nal, &nalBs) + && h264::NAL_UNIT_TYPE_SPS == nal.type) + { + _outSps = {}; + h264::read_sps(&_outSps, &nalBs); + + return true; + } + + bs.p = nalStart + nalSize; + bs.bits_left = 8; + } + + return false; + } + inline bool parseParameterSets( h264::SPS* _spsArray , uint32_t _spsArrayLen @@ -264,7 +415,8 @@ namespace bgfx h264::SPS sps = {}; h264::read_sps(&sps, &nalBs); - if (sps.seq_parameter_set_id < (int)_spsArrayLen) + if (isSpsInRange(sps) + && sps.seq_parameter_set_id < (int)_spsArrayLen) { _spsArray[sps.seq_parameter_set_id] = sps; @@ -282,7 +434,9 @@ namespace bgfx h264::PPS pps = {}; h264::read_pps(&pps, &nalBs); - if (pps.pic_parameter_set_id < (int)_ppsArrayLen) + if (isInRange(pps.seq_parameter_set_id, 0, int32_t(_spsArrayLen) - 1) + && isPpsInRange(pps, _spsArray[pps.seq_parameter_set_id]) + && pps.pic_parameter_set_id < (int)_ppsArrayLen) { _ppsArray[pps.pic_parameter_set_id] = pps; @@ -411,6 +565,8 @@ namespace bgfx , h264::NALHeader& _outNal , h264::SliceHeader& _outSlice , bool& _outIsIdr + , const bool* _spsValid + , const bool* _ppsValid ) { h264::Bitstream bs; @@ -435,44 +591,38 @@ namespace bgfx return false; } - if (h264::NAL_UNIT_TYPE_SPS == nal.type) - { - h264::SPS sps = {}; - h264::read_sps(&sps, &nalBs); - - if (sps.seq_parameter_set_id < 32) - { - const_cast(_spsArray)[sps.seq_parameter_set_id] = sps; - } - - bs.p = nalStart + nalSize; - bs.bits_left = 8; - - continue; - } - - if (h264::NAL_UNIT_TYPE_PPS == nal.type) - { - h264::PPS pps = {}; - h264::read_pps(&pps, &nalBs); - - if (pps.pic_parameter_set_id < 256) - { - _ppsArray[pps.pic_parameter_set_id] = pps; - } - - bs.p = nalStart + nalSize; - bs.bits_left = 8; - - continue; - } - if (h264::NAL_UNIT_TYPE_CODED_SLICE_NON_IDR == nal.type || h264::NAL_UNIT_TYPE_CODED_SLICE_IDR == nal.type) { + h264::Bitstream peek = nalBs; + peek.ue(); + const uint32_t sliceType = peek.ue(); + const uint32_t ppsId = peek.ue(); + + if (uint32_t(h264::MAX_SLICE_TYPE) < sliceType + || uint32_t(h264::MAX_PPS_ID) < ppsId + || !_ppsValid[ppsId]) + { + return false; + } + + const uint32_t spsId = uint32_t(_ppsArray[ppsId].seq_parameter_set_id); + + if (uint32_t(h264::MAX_SPS_ID) < spsId + || !_spsValid[spsId]) + { + return false; + } + h264::SliceHeader sh = {}; h264::read_slice_header(&sh, &nal, _ppsArray, const_cast(_spsArray), &nalBs); + if (0 != sh.field_pic_flag + || !isSliceHeaderInRange(sh, _ppsArray[ppsId], _spsArray[spsId]) ) + { + return false; + } + _outNal = nal; _outSlice = sh; _outIsIdr = (h264::NAL_UNIT_TYPE_CODED_SLICE_IDR == nal.type); @@ -803,6 +953,8 @@ namespace bgfx return; } + dropOverSoftCap(); + if (m_head == m_items.size() && m_head > 0) { m_items.clear(); @@ -837,6 +989,44 @@ namespace bgfx } } + void dropOverSoftCap() + { + if (0 == m_softCapBytes) + { + return; + } + + size_t pendingBytes = 0; + + for (size_t ii = m_head; ii < m_items.size(); ++ii) + { + pendingBytes += m_items[ii].data.size(); + } + + while (pendingBytes > m_softCapBytes) + { + size_t nextIdr = m_head + 1; + + while (nextIdr < m_items.size() + && !m_items[nextIdr].isIdr) + { + ++nextIdr; + } + + if (nextIdr >= m_items.size() ) + { + break; + } + + for (size_t ii = m_head; ii < nextIdr; ++ii) + { + pendingBytes -= m_items[ii].data.size(); + } + + m_head = nextIdr; + } + } + void clear() { m_items.clear(); @@ -902,52 +1092,94 @@ namespace bgfx typedef AuCache AuQueue; - inline int64_t applyLoopWrap(int64_t _ptsUs, int64_t _minPts, int64_t _maxPts) + inline int64_t applyLoopWrap(int64_t _ptsUs, int64_t _minPts, int64_t _maxPts, uint32_t _numAus) { if (_minPts >= _maxPts - || _ptsUs <= _maxPts) + || _ptsUs <= _maxPts + || 2 > _numAus) { return _ptsUs; } - const int64_t span = _maxPts - _minPts + 1; + const int64_t duration = (_maxPts - _minPts) / int64_t(_numAus - 1); + const int64_t span = _maxPts - _minPts + duration; const int64_t wrapped = ( (_ptsUs - _minPts) % span) + _minPts; return wrapped; } template inline bool reorderPoolCovers( - const ReorderEntryT* _pool + const AuCache& _cache + , const ReorderEntryT* _pool , uint32_t _count , int64_t _ptsUs - , int64_t _toleranceUs ) { - for (uint32_t ii = 0; ii < _count; ++ii) - { - if (-1 != _pool[ii].displayOrder) - { - const int64_t diff = _pool[ii].ptsUs >= _ptsUs - ? _pool[ii].ptsUs - _ptsUs - : _ptsUs - _pool[ii].ptsUs - ; + int64_t targetPts = INT64_MIN; - if (diff <= _toleranceUs) - { - return true; - } + for (uint32_t ii = 0; ii < _cache.size(); ++ii) + { + const int64_t pts = _cache.at(ii).ptsUs; + + if (pts <= _ptsUs + && pts > targetPts) + { + targetPts = pts; } } + + if (INT64_MIN == targetPts) + { + return true; + } + + for (uint32_t ii = 0; ii < _count; ++ii) + { + if (-1 != _pool[ii].displayOrder + && targetPts == _pool[ii].ptsUs) + { + return true; + } + } + return false; } - template + template + inline bool hasReorderSpace(const ReorderEntryT* _pool, uint32_t _count) + { + for (uint32_t ii = 0; ii < _count; ++ii) + { + if (-1 == _pool[ii].displayOrder) + { + return true; + } + } + + return false; + } + + template + inline void releaseReorderBefore(ReorderEntryT* _pool, uint32_t _count, int64_t _ptsUs) + { + for (uint32_t ii = 0; ii < _count; ++ii) + { + if (-1 != _pool[ii].displayOrder + && _pool[ii].ptsUs < _ptsUs) + { + _pool[ii].displayOrder = -1; + } + } + } + + template inline bool faultInFromCache( - AuCache& _cache - , int64_t _ptsUs - , ResetFn _reset - , DecodeOneFn _decodeOne - , HasSpaceFn _hasReorderSpace + AuCache& _cache + , int64_t _ptsUs + , ResetFn _reset + , DecodeOneFn _decodeOne + , ReorderEntryT* _pool + , uint32_t _count ) { if (!_cache.isRetaining() @@ -966,38 +1198,28 @@ namespace bgfx _reset(); _cache.rewindTo(uint32_t(idx) ); + int64_t bestPts = INT64_MIN; + while (!_cache.empty() - && _hasReorderSpace() ) + && hasReorderSpace(_pool, _count) ) { const PendingAU& au = _cache.front(); - _decodeOne(au.data.data(), uint32_t(au.data.size() ), au.ptsUs); + const int64_t pts = au.ptsUs; - const bool reachedTarget = au.ptsUs >= _ptsUs; + _decodeOne(au.data.data(), uint32_t(au.data.size() ), pts); _cache.pop(); - if (reachedTarget) + if (pts <= _ptsUs + && pts > bestPts) { - break; + bestPts = pts; + releaseReorderBefore(_pool, _count, bestPts); } } return true; } - template - inline bool hasReorderSpace(const ReorderEntryT* _pool, uint32_t _count) - { - for (uint32_t ii = 0; ii < _count; ++ii) - { - if (-1 == _pool[ii].displayOrder) - { - return true; - } - } - - return false; - } - template inline uint32_t pickReorderSlotOrEvictOldest(const ReorderEntryT* _pool, uint32_t _count) { @@ -1093,154 +1315,75 @@ namespace bgfx , DispatchFn _dispatch ) { - if (-1 == _displayedSlot) - { - int32_t bestSlot = -1; - int64_t minPts = INT64_MAX; - int64_t maxPts = INT64_MIN; - - for (uint32_t ii = 0; ii < _poolCount; ++ii) - { - if (-1 != _pool[ii].displayOrder) - { - if (_pool[ii].ptsUs < minPts) - { - minPts = _pool[ii].ptsUs; - bestSlot = int32_t(ii); - } - - if (_pool[ii].ptsUs > maxPts) - { - maxPts = _pool[ii].ptsUs; - } - } - } - - if (-1 == bestSlot) - { - return; - } - - if (minPts == maxPts - && !_streamDrained) - { - return; - } - - _displayedSlot = bestSlot; - _dispatch(uint32_t(_displayedSlot) ); - - return; - } - - const int64_t curPts = _pool[_displayedSlot].ptsUs; - - if (_presentationTimeUs + 500000 < curPts) - { - int32_t snapSlot = -1; - int64_t minPts = INT64_MAX; - for (uint32_t ii = 0; ii < _poolCount; ++ii) - { - if (-1 != _pool[ii].displayOrder - && _pool[ii].ptsUs < curPts - && _pool[ii].ptsUs < minPts) - { - minPts = _pool[ii].ptsUs; - snapSlot = int32_t(ii); - } - } - - if (-1 == snapSlot) - { - return; - } - - if (-1 != _prevDisplayedSlot) - { - _pool[_prevDisplayedSlot].displayOrder = -1; - } - - _prevDisplayedSlot = _displayedSlot; - _displayedSlot = snapSlot; - _dispatch(uint32_t(_displayedSlot) ); - - return; - } - - if (_presentationTimeUs > curPts + 500000) - { - int32_t jumpSlot = -1; - int64_t bestPts = curPts; - for (uint32_t ii = 0; ii < _poolCount; ++ii) - { - if (-1 != _pool[ii].displayOrder - && _pool[ii].ptsUs > bestPts - && _pool[ii].ptsUs <= _presentationTimeUs) - { - bestPts = _pool[ii].ptsUs; - jumpSlot = int32_t(ii); - } - } - - if (-1 != jumpSlot) - { - if (-1 != _prevDisplayedSlot) - { - _pool[_prevDisplayedSlot].displayOrder = -1; - } - - _prevDisplayedSlot = _displayedSlot; - _displayedSlot = jumpSlot; - _dispatch(uint32_t(_displayedSlot) ); - - return; - } - } - - int32_t nextSlot = -1; - int64_t nextPts = INT64_MAX; + int32_t minSlot = -1; + int64_t minPts = INT64_MAX; int64_t maxPts = INT64_MIN; for (uint32_t ii = 0; ii < _poolCount; ++ii) { if (-1 != _pool[ii].displayOrder) { - if (_pool[ii].ptsUs > maxPts) + if (_pool[ii].ptsUs < minPts) { - maxPts = _pool[ii].ptsUs; + minPts = _pool[ii].ptsUs; + minSlot = int32_t(ii); } - if (_pool[ii].ptsUs > curPts - && _pool[ii].ptsUs < nextPts) - { - nextPts = _pool[ii].ptsUs; - nextSlot = int32_t(ii); - } + maxPts = bx::max(maxPts, _pool[ii].ptsUs); } } - if (-1 == nextSlot) + if (-1 == minSlot) { return; } - if (_presentationTimeUs < nextPts) + int32_t pickSlot = -1; + int64_t pickPts = INT64_MIN; + + for (uint32_t ii = 0; ii < _poolCount; ++ii) + { + const int64_t pts = _pool[ii].ptsUs; + + if (-1 != _pool[ii].displayOrder + && pts > pickPts + && pts <= _presentationTimeUs + && (pts < maxPts || _streamDrained) ) + { + pickPts = pts; + pickSlot = int32_t(ii); + } + } + + if (-1 == pickSlot) + { + if (-1 != _displayedSlot + || (minPts == maxPts && !_streamDrained) ) + { + return; + } + + pickPts = minPts; + pickSlot = minSlot; + } + + if (pickSlot == _displayedSlot) { return; } - if (nextPts == maxPts && !_streamDrained) + for (uint32_t ii = 0; ii < _poolCount; ++ii) { - return; - } - - if (-1 != _prevDisplayedSlot) - { - _pool[_prevDisplayedSlot].displayOrder = -1; + if (int32_t(ii) != _displayedSlot + && -1 != _pool[ii].displayOrder + && _pool[ii].ptsUs < pickPts) + { + _pool[ii].displayOrder = -1; + } } _prevDisplayedSlot = _displayedSlot; - _displayedSlot = nextSlot; + _displayedSlot = pickSlot; _dispatch(uint32_t(_displayedSlot) ); } } diff --git a/src/video_d3d11.cpp b/src/video_d3d11.cpp index 5854bfde0..f17d577c5 100644 --- a/src/video_d3d11.cpp +++ b/src/video_d3d11.cpp @@ -58,7 +58,6 @@ namespace bgfx { namespace d3d11 , m_videoDevice(NULL) , m_videoContext(NULL) , m_decoder(NULL) - , m_sampler(NULL) , m_codedWidth(0) , m_codedHeight(0) , m_dstWidth(0) @@ -84,9 +83,12 @@ namespace bgfx { namespace d3d11 bx::memSet(m_dpbState, 0, sizeof(m_dpbState) ); bx::memSet(m_referenceUsage, 0, sizeof(m_referenceUsage) ); bx::memSet(&m_spsActive, 0, sizeof(m_spsActive) ); + bx::memSet(m_spsValid, 0, sizeof(m_spsValid) ); + bx::memSet(m_ppsValid, 0, sizeof(m_ppsValid) ); bx::memSet(m_spsArray, 0, sizeof(m_spsArray) ); bx::memSet(m_ppsArray, 0, sizeof(m_ppsArray) ); bx::memSet(m_yuvToRgb, 0, sizeof(m_yuvToRgb) ); + bx::memSet(&m_picture, 0, sizeof(m_picture) ); for (uint32_t ii = 0; ii < kVideoMaxReorderInFlight; ++ii) { @@ -122,7 +124,7 @@ namespace bgfx { namespace d3d11 h264::PPS ppsActive = {}; - if (!parseParameterSets(m_spsArray, BX_COUNTOF(m_spsArray), m_ppsArray, BX_COUNTOF(m_ppsArray), m_spsActive, ppsActive, _init.parameterSets, _init.parameterSetsSize) ) + if (!parseParameterSets(m_spsArray, BX_COUNTOF(m_spsArray), m_ppsArray, BX_COUNTOF(m_ppsArray), m_spsActive, ppsActive, _init.parameterSets, _init.parameterSetsSize, m_spsValid, m_ppsValid) ) { BX_TRACE("VideoDecoderD3D11: failed to parse SPS/PPS from parameter sets."); return false; @@ -137,6 +139,7 @@ namespace bgfx { namespace d3d11 } getYuvToRgb(m_yuvToRgb, m_spsActive); + m_picture = getPictureRect(m_spsActive); m_dstWidth = _width; m_dstHeight = _height; @@ -290,8 +293,8 @@ namespace bgfx { namespace d3d11 } { - const uint32_t reorderW = (m_dstWidth + 1) & ~1u; - const uint32_t reorderH = (m_dstHeight + 1) & ~1u; + const uint32_t reorderW = m_picture.width; + const uint32_t reorderH = m_picture.height; for (uint32_t ii = 0; ii < kVideoMaxReorderInFlight; ++ii) { @@ -373,18 +376,6 @@ namespace bgfx { namespace d3d11 } } - m_sampler = videoGetSamplerState( - _renderer - , BGFX_SAMPLER_U_CLAMP - | BGFX_SAMPLER_V_CLAMP - ); - - if (NULL == m_sampler) - { - BX_TRACE("VideoDecoderD3D11: videoGetSamplerState failed."); - return false; - } - BX_TRACE("VideoDecoderD3D11: H.264 decoder ready, coded %dx%d (padded %ux%u), dst %dx%d." , m_codedWidth, m_codedHeight , paddedWidth, paddedHeight @@ -413,8 +404,6 @@ namespace bgfx { namespace d3d11 DX_RELEASE_I(m_videoContext); DX_RELEASE_I(m_videoDevice); - m_sampler = NULL; - m_currentSlot = 0; m_nextSlot = 0; m_numActiveRefs = 0; @@ -475,9 +464,9 @@ namespace bgfx { namespace d3d11 && m_auQueue.isRetaining() && m_auQueue.minPts() < m_auQueue.maxPts() ) { - adjustedClock = applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts() ); + adjustedClock = applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts(), m_auQueue.size() ); - const bool poolCovers = reorderPoolCovers(m_reorderPool, kVideoMaxReorderInFlight, adjustedClock, 500000); + const bool poolCovers = reorderPoolCovers(m_auQueue, m_reorderPool, kVideoMaxReorderInFlight, adjustedClock); const bool forwardCanCatchUp = true && !m_auQueue.empty() && m_auQueue.front().ptsUs <= adjustedClock + 500000 @@ -494,10 +483,8 @@ namespace bgfx { namespace d3d11 { decodeOneAU(_bs, _bsSize, _pts); } - , [this]() - { - return hasReorderSpace(m_reorderPool, kVideoMaxReorderInFlight); - } + , m_reorderPool + , kVideoMaxReorderInFlight ); } } @@ -548,7 +535,7 @@ namespace bgfx { namespace d3d11 h264::SliceHeader sh = {}; bool isIdr = false; - if (!parseSliceFromAccessUnit(m_spsArray, m_ppsArray, _bitstream, _bitstreamSize, nal, sh, isIdr) ) + if (!parseSliceFromAccessUnit(m_spsArray, m_ppsArray, _bitstream, _bitstreamSize, nal, sh, isIdr, m_spsValid, m_ppsValid) ) { BX_TRACE("VideoDecoderD3D11: no slice NAL found in access unit."); return false; @@ -607,9 +594,9 @@ namespace bgfx { namespace d3d11 DXVA_Qmatrix_H264 qmatrix = {}; picParams.wFrameWidthInMbsMinus1 = uint16_t(sps.pic_width_in_mbs_minus1); - picParams.wFrameHeightInMbsMinus1 = uint16_t(sps.pic_height_in_map_units_minus1); + picParams.wFrameHeightInMbsMinus1 = uint16_t( (sps.pic_height_in_map_units_minus1 + 1) * (0 != sps.frame_mbs_only_flag ? 1 : 2) - 1); picParams.IntraPicFlag = isIdr ? 1 : 0; - picParams.MbaffFrameFlag = 0; + picParams.MbaffFrameFlag = 0 != sps.mb_adaptive_frame_field_flag ? 1 : 0; picParams.field_pic_flag = 0; picParams.chroma_format_idc = 1; picParams.bit_depth_chroma_minus8 = uint8_t(sps.bit_depth_chroma_minus8); @@ -867,20 +854,20 @@ namespace bgfx { namespace d3d11 } { - const uint32_t copyW = (m_dstWidth + 1) & ~1u; - const uint32_t copyH = (m_dstHeight + 1) & ~1u; - const uint32_t srcW = bx::min(copyW, m_codedWidth); - const uint32_t srcH = bx::min(copyH, m_codedHeight); + const uint32_t srcX = m_picture.x; + const uint32_t srcY = m_picture.y; + const uint32_t srcW = m_picture.width; + const uint32_t srcH = m_picture.height; ID3D11DeviceContext* deviceCtx = s_videoD3D11.deviceCtx; const D3D11_BOX boxY = { - .left = 0, - .top = 0, + .left = srcX, + .top = srcY, .front = 0, - .right = srcW, - .bottom = srcH, + .right = srcX + srcW, + .bottom = srcY + srcH, .back = 1, }; deviceCtx->CopySubresourceRegion( @@ -892,11 +879,11 @@ namespace bgfx { namespace d3d11 const D3D11_BOX boxUV = { - .left = 0, - .top = 0, + .left = srcX / 2, + .top = srcY / 2, .front = 0, - .right = srcW / 2, - .bottom = srcH / 2, + .right = (srcX + srcW) / 2, + .bottom = (srcY + srcH) / 2, .back = 1, }; deviceCtx->CopySubresourceRegion( @@ -990,7 +977,13 @@ namespace bgfx { namespace d3d11 ID3D11ShaderResourceView* srvs[3] = { NULL, m_reorderSrvY[_poolIdx], m_reorderSrvCbCr[_poolIdx] }; deviceCtx->CSSetShaderResources(0, 3, srvs); - ID3D11SamplerState* samplers[3] = { NULL, m_sampler, m_sampler }; + ID3D11SamplerState* sampler = videoGetSamplerState( + m_renderer + , BGFX_SAMPLER_U_CLAMP + | BGFX_SAMPLER_V_CLAMP + ); + + ID3D11SamplerState* samplers[3] = { NULL, sampler, sampler }; deviceCtx->CSSetSamplers(0, 3, samplers); ID3D11UnorderedAccessView* uav = _dst.m_uav; @@ -1037,7 +1030,6 @@ namespace bgfx { namespace d3d11 ID3D11Texture2D* m_reorderTex[kVideoMaxReorderInFlight]; ID3D11ShaderResourceView* m_reorderSrvY[kVideoMaxReorderInFlight]; ID3D11ShaderResourceView* m_reorderSrvCbCr[kVideoMaxReorderInFlight]; - ID3D11SamplerState* m_sampler; DpbState m_dpbState[kNumDpbSlots]; uint8_t m_referenceUsage[kNumDpbSlots]; @@ -1058,7 +1050,8 @@ namespace bgfx { namespace d3d11 int32_t m_displayedSlot; int32_t m_prevDisplayedSlot; - float m_yuvToRgb[12]; + float m_yuvToRgb[12]; + VideoPictureRect m_picture; uint32_t m_statusReportFeedback; @@ -1070,6 +1063,8 @@ namespace bgfx { namespace d3d11 h264::SPS m_spsActive; h264::SPS m_spsArray[kVideoMaxH264SpsCount]; h264::PPS m_ppsArray[kVideoMaxH264PpsCount]; + bool m_spsValid[kVideoMaxH264SpsCount]; + bool m_ppsValid[kVideoMaxH264PpsCount]; }; VideoDecoderD3D11* videoDecoderCreate(const VideoDecoderInit& _init, RendererContextD3D11* _renderer, uint16_t _width, uint16_t _height) diff --git a/src/video_d3d12.cpp b/src/video_d3d12.cpp index 6540e7cbb..91a6b75b3 100644 --- a/src/video_d3d12.cpp +++ b/src/video_d3d12.cpp @@ -109,6 +109,9 @@ namespace bgfx { namespace d3d12 bx::memSet(m_dpbState, 0, sizeof(m_dpbState) ); bx::memSet(m_referenceUsage, 0, sizeof(m_referenceUsage) ); bx::memSet(&m_spsActive, 0, sizeof(m_spsActive) ); + bx::memSet(m_spsValid, 0, sizeof(m_spsValid) ); + bx::memSet(m_ppsValid, 0, sizeof(m_ppsValid) ); + bx::memSet(&m_picture, 0, sizeof(m_picture) ); for (uint32_t ii = 0; ii < kBitstreamRingFrames; ++ii) { @@ -160,7 +163,7 @@ namespace bgfx { namespace d3d12 } h264::PPS ppsActive = {}; - if (!bgfx::parseParameterSets(m_spsArray, BX_COUNTOF(m_spsArray), m_ppsArray, BX_COUNTOF(m_ppsArray), m_spsActive, ppsActive, _init.parameterSets, _init.parameterSetsSize) ) + if (!bgfx::parseParameterSets(m_spsArray, BX_COUNTOF(m_spsArray), m_ppsArray, BX_COUNTOF(m_ppsArray), m_spsActive, ppsActive, _init.parameterSets, _init.parameterSetsSize, m_spsValid, m_ppsValid) ) { BX_TRACE("VideoDecoderD3D12: failed to parse SPS/PPS from parameter sets."); return false; @@ -187,6 +190,7 @@ namespace bgfx { namespace d3d12 m_codedWidth = (m_spsActive.pic_width_in_mbs_minus1 + 1) * 16; m_codedHeight = (m_spsActive.pic_height_in_map_units_minus1 + 1) * 16 * (m_spsActive.frame_mbs_only_flag ? 1 : 2); + m_picture = getPictureRect(m_spsActive); m_numDpbSlots = bx::clamp(uint32_t(m_spsActive.num_ref_frames) + 1, 2, kNumDpbSlots); const DXGI_FORMAT decodeFormat = DXGI_FORMAT_NV12; @@ -636,7 +640,8 @@ namespace bgfx { namespace d3d12 bool decode(const VideoDecoderFrame& _frame, TextureD3D12& _dst) { - if (NULL == m_decoder) + if (NULL == m_decoder + || FAILED(s_videoD3D12.device->GetDeviceRemovedReason() ) ) { return false; } @@ -677,9 +682,9 @@ namespace bgfx { namespace d3d12 && m_auQueue.isRetaining() && m_auQueue.minPts() < m_auQueue.maxPts() ) { - adjustedClock = bgfx::applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts() ); + adjustedClock = bgfx::applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts(), m_auQueue.size() ); - const bool poolCovers = bgfx::reorderPoolCovers(m_reorderPool, kVideoMaxReorderInFlight, adjustedClock, 500000); + const bool poolCovers = bgfx::reorderPoolCovers(m_auQueue, m_reorderPool, kVideoMaxReorderInFlight, adjustedClock); const bool forwardCanCatchUp = true && !m_auQueue.empty() && m_auQueue.front().ptsUs <= adjustedClock + 500000 @@ -696,10 +701,8 @@ namespace bgfx { namespace d3d12 { decodeOneAU(_bs, _bsSize, _pts); } - , [this]() - { - return bgfx::hasReorderSpace(m_reorderPool, kVideoMaxReorderInFlight); - } + , m_reorderPool + , kVideoMaxReorderInFlight ); } } @@ -750,7 +753,7 @@ namespace bgfx { namespace d3d12 h264::NALHeader nal = {}; h264::SliceHeader sh = {}; bool isIdr = false; - if (!bgfx::parseSliceFromAccessUnit(m_spsArray, m_ppsArray, _bitstream, _bitstreamSize, nal, sh, isIdr) ) + if (!bgfx::parseSliceFromAccessUnit(m_spsArray, m_ppsArray, _bitstream, _bitstreamSize, nal, sh, isIdr, m_spsValid, m_ppsValid) ) { BX_TRACE("VideoDecoderD3D12: no slice NAL found in access unit."); return false; @@ -820,10 +823,17 @@ namespace bgfx { namespace d3d12 HRESULT hr = m_videoCmdAllocator[allocIdx]->Reset(); BX_WARN(SUCCEEDED(hr), "VideoDecoderD3D12: video allocator Reset failed (0x%08x).", hr); - BX_UNUSED(hr); - hr = m_videoCmd->Reset(m_videoCmdAllocator[allocIdx]); - BX_WARN(SUCCEEDED(hr), "VideoDecoderD3D12: video cmdlist Reset failed (0x%08x).", hr); + if (SUCCEEDED(hr) ) + { + hr = m_videoCmd->Reset(m_videoCmdAllocator[allocIdx]); + BX_WARN(SUCCEEDED(hr), "VideoDecoderD3D12: video cmdlist Reset failed (0x%08x).", hr); + } + + if (FAILED(hr) ) + { + return false; + } if (m_dpbState[m_currentSlot].state != D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE) { @@ -878,9 +888,9 @@ namespace bgfx { namespace d3d12 DXVA_Slice_H264_Short sliceCtl = {}; picParams.wFrameWidthInMbsMinus1 = uint16_t(sps.pic_width_in_mbs_minus1); - picParams.wFrameHeightInMbsMinus1 = uint16_t(sps.pic_height_in_map_units_minus1); + picParams.wFrameHeightInMbsMinus1 = uint16_t( (sps.pic_height_in_map_units_minus1 + 1) * (0 != sps.frame_mbs_only_flag ? 1 : 2) - 1); picParams.IntraPicFlag = isIdr ? 1 : 0; - picParams.MbaffFrameFlag = 0; + picParams.MbaffFrameFlag = 0 != sps.mb_adaptive_frame_field_flag ? 1 : 0; picParams.field_pic_flag = 0; picParams.chroma_format_idc = 1; picParams.bit_depth_chroma_minus8 = uint8_t(sps.bit_depth_chroma_minus8); @@ -1053,6 +1063,12 @@ namespace bgfx { namespace d3d12 { m_videoCmd->ResourceBarrier(numReferenceBarriers, referenceBarriers); } + if (FAILED(s_videoD3D12.device->GetDeviceRemovedReason() ) ) + { + m_videoCmd->Close(); + return false; + } + m_videoCmd->DecodeFrame(m_decoder, &output, &input); if (numReferenceBarriers > 0) { @@ -1218,8 +1234,16 @@ namespace bgfx { namespace d3d12 hr = m_copyCmdAllocator[copyRingIdx]->Reset(); BX_WARN(SUCCEEDED(hr), "VideoDecoderD3D12: copy allocator Reset failed (0x%08x).", hr); - hr = m_copyCmdList->Reset(m_copyCmdAllocator[copyRingIdx], NULL); - BX_WARN(SUCCEEDED(hr), "VideoDecoderD3D12: copy cmdlist Reset failed (0x%08x).", hr); + if (SUCCEEDED(hr) ) + { + hr = m_copyCmdList->Reset(m_copyCmdAllocator[copyRingIdx], NULL); + BX_WARN(SUCCEEDED(hr), "VideoDecoderD3D12: copy cmdlist Reset failed (0x%08x).", hr); + } + + if (FAILED(hr) ) + { + return false; + } ID3D12GraphicsCommandList* commandList = m_copyCmdList; @@ -1299,11 +1323,13 @@ namespace bgfx { namespace d3d12 } { - const uint32_t reorderW = (m_dstWidth + 1) & ~1u; - const uint32_t reorderH = (m_dstHeight + 1) & ~1u; + const uint32_t x0 = m_picture.x; + const uint32_t y0 = m_picture.y; + const uint32_t x1 = m_picture.x + m_picture.width; + const uint32_t y1 = m_picture.y + m_picture.height; - const D3D12_BOX lumaBox = { 0, 0, 0, reorderW, reorderH, 1 }; - const D3D12_BOX chromaBox = { 0, 0, 0, reorderW / 2, reorderH / 2, 1 }; + const D3D12_BOX lumaBox = { .left = x0, .top = y0, .front = 0, .right = x1, .bottom = y1, .back = 1 }; + const D3D12_BOX chromaBox = { .left = x0 / 2, .top = y0 / 2, .front = 0, .right = x1 / 2, .bottom = y1 / 2, .back = 1 }; { D3D12_TEXTURE_COPY_LOCATION dst @@ -1449,8 +1475,8 @@ namespace bgfx { namespace d3d12 { .Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D, .Alignment = 0, - .Width = (m_dstWidth + 1) & ~1u, - .Height = (m_dstHeight + 1) & ~1u, + .Width = m_picture.width, + .Height = m_picture.height, .DepthOrArraySize = 1, .MipLevels = 1, .Format = DXGI_FORMAT_NV12, @@ -1654,6 +1680,8 @@ namespace bgfx { namespace d3d12 uint64_t m_bitstreamCursor; ID3D12Resource* m_yuvToRgbCbv; + VideoPictureRect m_picture; + ID3D12Resource* m_dpb; ID3D12Resource* m_dpbOutput; bool m_referenceOnly; @@ -1687,6 +1715,8 @@ namespace bgfx { namespace d3d12 h264::SPS m_spsActive; h264::SPS m_spsArray[kVideoMaxH264SpsCount]; h264::PPS m_ppsArray[kVideoMaxH264PpsCount]; + bool m_spsValid[kVideoMaxH264SpsCount]; + bool m_ppsValid[kVideoMaxH264PpsCount]; }; VideoDecoderD3D12* videoDecoderCreate(const VideoDecoderInit& _init, RendererContextD3D12* _renderer, uint16_t _width, uint16_t _height) diff --git a/src/video_vk.cpp b/src/video_vk.cpp index e96045f46..bd6f65c10 100644 --- a/src/video_vk.cpp +++ b/src/video_vk.cpp @@ -133,6 +133,7 @@ namespace bgfx { namespace vk { bx::memSet(m_referenceUsage, 0, sizeof(m_referenceUsage) ); bx::memSet(&m_spsActive, 0, sizeof(m_spsActive) ); + bx::memSet(&m_picture, 0, sizeof(m_picture) ); bx::memSet(m_spsArray, 0, sizeof(m_spsArray) ); bx::memSet(m_ppsArray, 0, sizeof(m_ppsArray) ); bx::memSet(m_spsValid, 0, sizeof(m_spsValid) ); @@ -225,6 +226,7 @@ namespace bgfx { namespace vk m_dstHeight = _height; m_codedWidth = uint32_t(m_spsActive.pic_width_in_mbs_minus1 + 1) * 16; m_codedHeight = uint32_t(m_spsActive.pic_height_in_map_units_minus1 + 1) * 16 * (m_spsActive.frame_mbs_only_flag ? 1 : 2); + m_picture = getPictureRect(m_spsActive); m_numDpbSlots = bx::min(bx::max(uint32_t(m_spsActive.num_ref_frames) + 1, 2), kNumDpbSlots); VkDevice device = s_videoVK.device; @@ -1131,9 +1133,9 @@ namespace bgfx { namespace vk && m_auQueue.minPts() < m_auQueue.maxPts() ) { - adjustedClock = bgfx::applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts() ); + adjustedClock = bgfx::applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts(), m_auQueue.size() ); - const bool poolCovers = bgfx::reorderPoolCovers(m_reorderPool, kVideoMaxReorderInFlight, adjustedClock, 500000); + const bool poolCovers = bgfx::reorderPoolCovers(m_auQueue, m_reorderPool, kVideoMaxReorderInFlight, adjustedClock); const bool forwardCanCatchUp = true && !m_auQueue.empty() && m_auQueue.front().ptsUs <= adjustedClock + 500000 @@ -1150,10 +1152,8 @@ namespace bgfx { namespace vk { decodeOneAU(_bs, _bsSize, _pts); } - , [this]() - { - return bgfx::hasReorderSpace(m_reorderPool, kVideoMaxReorderInFlight); - } + , m_reorderPool + , kVideoMaxReorderInFlight ); } } @@ -1334,7 +1334,7 @@ namespace bgfx { namespace vk h264::NALHeader nal = {}; h264::SliceHeader sh = {}; bool isIdr = false; - if (!bgfx::parseSliceFromAccessUnit(m_spsArray, m_ppsArray, _bitstream, _bitstreamSize, nal, sh, isIdr) ) + if (!bgfx::parseSliceFromAccessUnit(m_spsArray, m_ppsArray, _bitstream, _bitstreamSize, nal, sh, isIdr, m_spsValid, m_ppsValid) ) { return false; } @@ -1924,19 +1924,21 @@ namespace bgfx { namespace vk VkImage srcImage = m_dpbCoincide ? m_dpbImage : m_decodeOutputImage; uint32_t srcLayer = m_dpbCoincide ? m_currentSlot : 0; - const uint32_t reorderW = (m_dstWidth + 1) & ~1u; - const uint32_t reorderH = (m_dstHeight + 1) & ~1u; + const uint32_t reorderW = m_picture.width; + const uint32_t reorderH = m_picture.height; VkImageCopy cpy { .srcSubresource = { .aspectMask = VK_IMAGE_ASPECT_PLANE_0_BIT, .mipLevel = 0, .baseArrayLayer = srcLayer, .layerCount = 1 }, - .srcOffset = {}, + .srcOffset = { .x = int32_t(m_picture.x), .y = int32_t(m_picture.y), .z = 0 }, .dstSubresource = { .aspectMask = VK_IMAGE_ASPECT_PLANE_0_BIT, .mipLevel = 0, .baseArrayLayer = 0, .layerCount = 1 }, .dstOffset = {}, .extent = { reorderW, reorderH, 1 }, }; vkCmdCopyImage(gfx, srcImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, reorder.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy); + cpy.srcOffset.x = int32_t(m_picture.x / 2); + cpy.srcOffset.y = int32_t(m_picture.y / 2); cpy.extent.width = reorderW / 2; cpy.extent.height = reorderH / 2; cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_PLANE_1_BIT; @@ -2030,7 +2032,7 @@ namespace bgfx { namespace vk .flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT, .imageType = VK_IMAGE_TYPE_2D, .format = VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, - .extent = { (m_dstWidth + 1) & ~1u, (m_dstHeight + 1) & ~1u, 1 }, + .extent = { m_picture.width, m_picture.height, 1 }, .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, @@ -2670,6 +2672,8 @@ namespace bgfx { namespace vk VkBuffer m_yuvToRgbBuffer; VkDeviceMemory m_yuvToRgbBufferMemory; + VideoPictureRect m_picture; + VkCommandPool m_videoCommandPool; VkCommandBuffer m_videoCommandBuffer; VkFence m_videoFence;