From cd84669abc242ec59770f27cb6dec5f43ad76c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Branimir=20Karad=C5=BEi=C4=87?= Date: Mon, 12 Oct 2026 21:20:21 -0700 Subject: [PATCH] Fixed Metal video decode presenting stale pictures, and added AV1 decode on Metal. (#4047) --- src/video.h | 69 +- src/video_mtl.cpp | 1525 +++++++++++++------ tools/texturev/texturev.cpp | 91 +- tools/texturev/video_player.cpp | 2434 ++++++++++++++++++++++++++++--- tools/texturev/video_player.h | 4 +- 5 files changed, 3433 insertions(+), 690 deletions(-) diff --git a/src/video.h b/src/video.h index 0680c4e1b..d399f8450 100644 --- a/src/video.h +++ b/src/video.h @@ -258,11 +258,6 @@ namespace bgfx return poc; } - inline bool isInRange(int32_t _value, int32_t _min, int32_t _max) - { - return _min <= _value && _value <= _max; - } - static constexpr int32_t kVideoMaxH264SizeInMbsMinus1 = 1023; inline bool isDecodableProfile(int32_t _profileIdc) @@ -278,17 +273,17 @@ namespace bgfx { return true && isDecodableProfile(_sps.profile_idc) - && isInRange(_sps.seq_parameter_set_id, 0, h264::MAX_SPS_ID) - && isInRange(_sps.chroma_format_idc, 0, h264::MAX_CHROMA_FORMAT_IDC) - && isInRange(_sps.bit_depth_luma_minus8, 0, h264::MAX_BIT_DEPTH_MINUS8) - && isInRange(_sps.bit_depth_chroma_minus8, 0, h264::MAX_BIT_DEPTH_MINUS8) - && isInRange(_sps.log2_max_frame_num_minus4, 0, h264::MAX_LOG2_MINUS4) - && isInRange(_sps.pic_order_cnt_type, 0, h264::MAX_PIC_ORDER_CNT_TYPE) - && isInRange(_sps.log2_max_pic_order_cnt_lsb_minus4, 0, h264::MAX_LOG2_MINUS4) - && isInRange(_sps.num_ref_frames_in_pic_order_cnt_cycle, 0, h264::MAX_NUM_REF_FRAMES_IN_PIC_ORDER_CNT_CYCLE) - && isInRange(_sps.num_ref_frames, 0, h264::MAX_NUM_REF_FRAMES) - && isInRange(_sps.pic_width_in_mbs_minus1, 0, kVideoMaxH264SizeInMbsMinus1) - && isInRange(_sps.pic_height_in_map_units_minus1, 0, kVideoMaxH264SizeInMbsMinus1) + && bx::isInRange(_sps.seq_parameter_set_id, 0, h264::MAX_SPS_ID) + && bx::isInRange(_sps.chroma_format_idc, 0, h264::MAX_CHROMA_FORMAT_IDC) + && bx::isInRange(_sps.bit_depth_luma_minus8, 0, h264::MAX_BIT_DEPTH_MINUS8) + && bx::isInRange(_sps.bit_depth_chroma_minus8, 0, h264::MAX_BIT_DEPTH_MINUS8) + && bx::isInRange(_sps.log2_max_frame_num_minus4, 0, h264::MAX_LOG2_MINUS4) + && bx::isInRange(_sps.pic_order_cnt_type, 0, h264::MAX_PIC_ORDER_CNT_TYPE) + && bx::isInRange(_sps.log2_max_pic_order_cnt_lsb_minus4, 0, h264::MAX_LOG2_MINUS4) + && bx::isInRange(_sps.num_ref_frames_in_pic_order_cnt_cycle, 0, h264::MAX_NUM_REF_FRAMES_IN_PIC_ORDER_CNT_CYCLE) + && bx::isInRange(_sps.num_ref_frames, 0, h264::MAX_NUM_REF_FRAMES) + && bx::isInRange(_sps.pic_width_in_mbs_minus1, 0, kVideoMaxH264SizeInMbsMinus1) + && bx::isInRange(_sps.pic_height_in_map_units_minus1, 0, kVideoMaxH264SizeInMbsMinus1) ; } @@ -298,16 +293,16 @@ namespace bgfx const int32_t maxQpInit = h264::MAX_QP - h264::PIC_INIT_QP; return true - && isInRange(_pps.pic_parameter_set_id, 0, h264::MAX_PPS_ID) - && isInRange(_pps.seq_parameter_set_id, 0, h264::MAX_SPS_ID) + && bx::isInRange(_pps.pic_parameter_set_id, 0, h264::MAX_PPS_ID) + && bx::isInRange(_pps.seq_parameter_set_id, 0, h264::MAX_SPS_ID) && 0 == _pps.num_slice_groups_minus1 - && isInRange(_pps.num_ref_idx_l0_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) - && isInRange(_pps.num_ref_idx_l1_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) - && isInRange(_pps.weighted_bipred_idc, 0, h264::MAX_WEIGHTED_BIPRED_IDC) - && isInRange(_pps.pic_init_qp_minus26, -h264::PIC_INIT_QP - qpBdOffset, maxQpInit) - && isInRange(_pps.pic_init_qs_minus26, -h264::PIC_INIT_QP, maxQpInit) - && isInRange(_pps.chroma_qp_index_offset, -h264::MAX_CHROMA_QP_INDEX_OFFSET, h264::MAX_CHROMA_QP_INDEX_OFFSET) - && isInRange(_pps.second_chroma_qp_index_offset, -h264::MAX_CHROMA_QP_INDEX_OFFSET, h264::MAX_CHROMA_QP_INDEX_OFFSET) + && bx::isInRange(_pps.num_ref_idx_l0_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && bx::isInRange(_pps.num_ref_idx_l1_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && bx::isInRange(_pps.weighted_bipred_idc, 0, h264::MAX_WEIGHTED_BIPRED_IDC) + && bx::isInRange(_pps.pic_init_qp_minus26, -h264::PIC_INIT_QP - qpBdOffset, maxQpInit) + && bx::isInRange(_pps.pic_init_qs_minus26, -h264::PIC_INIT_QP, maxQpInit) + && bx::isInRange(_pps.chroma_qp_index_offset, -h264::MAX_CHROMA_QP_INDEX_OFFSET, h264::MAX_CHROMA_QP_INDEX_OFFSET) + && bx::isInRange(_pps.second_chroma_qp_index_offset, -h264::MAX_CHROMA_QP_INDEX_OFFSET, h264::MAX_CHROMA_QP_INDEX_OFFSET) ; } @@ -320,17 +315,17 @@ namespace bgfx const int32_t mbaff = 0 != _sps.mb_adaptive_frame_field_flag && 0 == _sh.field_pic_flag ? 1 : 0; return true - && isInRange(_sh.first_mb_in_slice * (1 + mbaff), 0, picSizeInMbs - 1) - && isInRange(_sh.slice_type, 0, h264::MAX_SLICE_TYPE) - && isInRange(sliceQp, -qpBdOffset, h264::MAX_QP) - && isInRange(_sh.num_ref_idx_l0_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) - && isInRange(_sh.num_ref_idx_l1_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) - && isInRange(_sh.cabac_init_idc, 0, h264::MAX_CABAC_INIT_IDC) - && isInRange(_sh.disable_deblocking_filter_idc, 0, h264::MAX_DISABLE_DEBLOCKING_FILTER_IDC) - && isInRange(_sh.slice_alpha_c0_offset_div2, -h264::MAX_DEBLOCKING_OFFSET_DIV2, h264::MAX_DEBLOCKING_OFFSET_DIV2) - && isInRange(_sh.slice_beta_offset_div2, -h264::MAX_DEBLOCKING_OFFSET_DIV2, h264::MAX_DEBLOCKING_OFFSET_DIV2) - && isInRange(_sh.idr_pic_id, 0, h264::MAX_IDR_PIC_ID) - && isInRange(_sh.redundant_pic_cnt, 0, h264::MAX_REDUNDANT_PIC_CNT) + && bx::isInRange(_sh.first_mb_in_slice * (1 + mbaff), 0, picSizeInMbs - 1) + && bx::isInRange(_sh.slice_type, 0, h264::MAX_SLICE_TYPE) + && bx::isInRange(sliceQp, -qpBdOffset, h264::MAX_QP) + && bx::isInRange(_sh.num_ref_idx_l0_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && bx::isInRange(_sh.num_ref_idx_l1_active_minus1, 0, h264::MAX_NUM_REF_IDX_ACTIVE_MINUS1) + && bx::isInRange(_sh.cabac_init_idc, 0, h264::MAX_CABAC_INIT_IDC) + && bx::isInRange(_sh.disable_deblocking_filter_idc, 0, h264::MAX_DISABLE_DEBLOCKING_FILTER_IDC) + && bx::isInRange(_sh.slice_alpha_c0_offset_div2, -h264::MAX_DEBLOCKING_OFFSET_DIV2, h264::MAX_DEBLOCKING_OFFSET_DIV2) + && bx::isInRange(_sh.slice_beta_offset_div2, -h264::MAX_DEBLOCKING_OFFSET_DIV2, h264::MAX_DEBLOCKING_OFFSET_DIV2) + && bx::isInRange(_sh.idr_pic_id, 0, h264::MAX_IDR_PIC_ID) + && bx::isInRange(_sh.redundant_pic_cnt, 0, h264::MAX_REDUNDANT_PIC_CNT) ; } @@ -434,7 +429,7 @@ namespace bgfx h264::PPS pps = {}; h264::read_pps(&pps, &nalBs); - if (isInRange(pps.seq_parameter_set_id, 0, int32_t(_spsArrayLen) - 1) + if (bx::isInRange(pps.seq_parameter_set_id, 0, int32_t(_spsArrayLen) - 1) && isPpsInRange(pps, _spsArray[pps.seq_parameter_set_id]) && pps.pic_parameter_set_id < (int)_ppsArrayLen) { diff --git a/src/video_mtl.cpp b/src/video_mtl.cpp index 55c560fd4..596a23276 100644 --- a/src/video_mtl.cpp +++ b/src/video_mtl.cpp @@ -15,8 +15,106 @@ # include # include +# if defined(__MAC_14_0) || defined(__IPHONE_17_0) +# define BGFX_VIDEO_METAL_AV1 1 +# else +# define BGFX_VIDEO_METAL_AV1 0 +# endif // defined(__MAC_14_0) || defined(__IPHONE_17_0) + namespace bgfx { namespace mtl { + namespace hevc + { + enum NalUnitType + { + BLA_W_LP = 16, + BLA_W_RADL = 17, + BLA_N_LP = 18, + IDR_W_RADL = 19, + IDR_N_LP = 20, + CRA_NUT = 21, + VPS_NUT = 32, + SPS_NUT = 33, + PPS_NUT = 34, + }; + + static constexpr uint32_t kNalHeaderSize = 2; + static constexpr uint32_t kNalUnitTypeShift = 1; + static constexpr uint8_t kNalUnitTypeMask = 0x3f; + + inline NalUnitType getNalUnitType(const uint8_t* _nal) + { + return NalUnitType( (_nal[0] >> kNalUnitTypeShift) & kNalUnitTypeMask); + } + + inline bool isIrap(NalUnitType _type) + { + return bx::isInRange(_type, BLA_W_LP, CRA_NUT); + } + } + + namespace h273 + { + enum ColourPrimaries + { + CP_BT709 = 1, + CP_UNSPECIFIED = 2, + }; + + enum TransferCharacteristics + { + TC_UNSPECIFIED = 2, + TC_SRGB = 13, + }; + + enum MatrixCoefficients + { + MC_IDENTITY = 0, + MC_BT709 = 1, + MC_UNSPECIFIED = 2, + MC_BT601 = 6, + MC_SMPTE240M = 7, + MC_BT2020_NCL = 9, + }; + } + + namespace av1 + { + enum ObuType + { + OBU_SEQUENCE_HEADER = 1, + OBU_TEMPORAL_DELIMITER = 2, + OBU_FRAME_HEADER = 3, + OBU_FRAME = 6, + }; + + enum FrameType + { + KEY_FRAME = 0, + }; + + enum Profile + { + PROFILE_MAIN = 0, + PROFILE_HIGH = 1, + PROFILE_PROFESSIONAL = 2, + }; + + static constexpr uint32_t kObuTypeShift = 3; + static constexpr uint8_t kObuTypeMask = 0xf; + static constexpr uint8_t kObuExtensionFlag = 0x4; + static constexpr uint8_t kObuHasSizeField = 0x2; + static constexpr uint32_t kObuHeaderMaxSize = 8; + static constexpr uint32_t kMaxLeadingZeros = 32; + static constexpr uint8_t kLeb128ValueMask = 0x7f; + static constexpr uint8_t kLeb128ContinuationBit = 0x80; + static constexpr uint8_t kAv1CMarkerVersion = 0x81; + } + + static constexpr uint32_t kNalLengthSize = 4; + static constexpr uint32_t kMaxParameterSets = 16; + static constexpr uint8_t kH264NalUnitTypeMask = 0x1f; + struct RendererContextMtl; void initVideoDecoder(); @@ -25,7 +123,7 @@ namespace bgfx { namespace mtl MTL::CommandBuffer* videoEnsureCommandBuffer(RendererContextMtl* _renderer); MTL::SamplerState* videoGetSamplerState(RendererContextMtl* _renderer, uint64_t _samplerFlags); - static void freeAvccBlockBxAlloc(void* _refcon, void* _memoryBlock, size_t _sizeInBytes) + static void freeBlockBxAlloc(void* _refcon, void* _memoryBlock, size_t _sizeInBytes) { BX_UNUSED(_refcon, _sizeInBytes); @@ -38,23 +136,23 @@ namespace bgfx { namespace mtl template static uint32_t visitAnnexBNalUnits(const uint8_t* _data, uint32_t _size, Fn _visit) { - uint32_t count = 0; - uint32_t pos = 0; + uint32_t count = 0; + uint32_t pos = 0; uint32_t nalStart = UINT32_MAX; while (pos + 3 < _size) { const bool sc4 = true - && 0 == _data[pos ] - && 0 == _data[pos+1] - && 0 == _data[pos+2] - && 1 == _data[pos+3] - ; + && 0 == _data[pos ] + && 0 == _data[pos+1] + && 0 == _data[pos+2] + && 1 == _data[pos+3] + ; const bool sc3 = true - && 0 == _data[pos] - && 0 == _data[pos+1] - && 1 == _data[pos+2] - ; + && 0 == _data[pos ] + && 0 == _data[pos+1] + && 1 == _data[pos+2] + ; if (sc3 || sc4) @@ -76,7 +174,8 @@ namespace bgfx { namespace mtl ++pos; } - if (UINT32_MAX != nalStart && nalStart < _size) + if (UINT32_MAX != nalStart + && nalStart < _size) { _visit(_data + nalStart, _size - nalStart); ++count; @@ -85,15 +184,15 @@ namespace bgfx { namespace mtl return count; } - static uint32_t annexBToAvcc(uint8_t* _out, const uint8_t* _data, uint32_t _size) + static uint32_t annexBToLengthPrefixed(uint8_t* _out, const uint8_t* _data, uint32_t _size) { uint32_t outPos = 0; visitAnnexBNalUnits(_data, _size, [&](const uint8_t* _nal, uint32_t _nalSize) { const uint32_t lengthBE = bx::endianSwap(_nalSize); - bx::memCopy(_out + outPos, &lengthBE, 4); - outPos += 4; + bx::memCopy(_out + outPos, &lengthBE, kNalLengthSize); + outPos += kNalLengthSize; bx::memCopy(_out + outPos, _nal, _nalSize); outPos += _nalSize; }); @@ -101,6 +200,21 @@ namespace bgfx { namespace mtl return outPos; } + static bool peekHevcAuIsIrap(const uint8_t* _data, uint32_t _size) + { + bool irap = false; + + visitAnnexBNalUnits(_data, _size, [&](const uint8_t* _nal, uint32_t _nalSize) + { + if (hevc::kNalHeaderSize <= _nalSize) + { + irap |= hevc::isIrap(hevc::getNalUnitType(_nal) ); + } + }); + + return irap; + } + static uint8_t getMatrixCoefficients(CMVideoFormatDescriptionRef _formatDesc) { CFTypeRef matrix = CMFormatDescriptionGetExtension(_formatDesc, kCMFormatDescriptionExtension_YCbCrMatrix); @@ -109,152 +223,483 @@ namespace bgfx { namespace mtl { if (CFEqual(matrix, kCMFormatDescriptionYCbCrMatrix_ITU_R_601_4) ) { - return 6; + return h273::MC_BT601; } if (CFEqual(matrix, kCMFormatDescriptionYCbCrMatrix_SMPTE_240M_1995) ) { - return 7; + return h273::MC_SMPTE240M; } if (CFEqual(matrix, kCMFormatDescriptionYCbCrMatrix_ITU_R_2020) ) { - return 9; + return h273::MC_BT2020_NCL; } } - return 1; + return h273::MC_BT709; + } + + struct Av1BitReader + { + Av1BitReader(const uint8_t* _data, uint32_t _size) + : data(_data) + , size(_size) + , pos(0) + { + } + + uint32_t read(uint32_t _bits) + { + uint32_t value = 0; + + for (uint32_t ii = 0; ii < _bits; ++ii) + { + const uint32_t byte = pos >> 3; + const uint32_t bit = byte < size + ? (data[byte] >> (7 - (pos & 7) ) ) & 1 + : 0 + ; + + value = (value << 1) | bit; + ++pos; + } + + return value; + } + + uint32_t uvlc() + { + uint32_t leadingZeros = 0; + + while (leadingZeros < av1::kMaxLeadingZeros + && 0 == read(1) ) + { + ++leadingZeros; + } + + if (av1::kMaxLeadingZeros <= leadingZeros) + { + return UINT32_MAX; + } + + return read(leadingZeros) + (1u << leadingZeros) - 1; + } + + bool overrun() const + { + return pos > size*8; + } + + const uint8_t* data; + uint32_t size; + uint32_t pos; + }; + + struct Av1SequenceHeader + { + uint32_t maxWidth; + uint32_t maxHeight; + uint8_t profile; + uint8_t levelIdx; + uint8_t tier; + uint8_t highBitdepth; + uint8_t twelveBit; + uint8_t monochrome; + uint8_t subsamplingX; + uint8_t subsamplingY; + uint8_t chromaSamplePosition; + uint8_t reducedStillPictureHeader; + uint8_t matrixCoefficients; + uint8_t colorRange; + }; + + static bool parseAv1SequenceHeader(const uint8_t* _data, uint32_t _size, Av1SequenceHeader& _out) + { + Av1BitReader br(_data, _size); + + _out = {}; + _out.profile = uint8_t(br.read(3) ); + br.read(1); + _out.reducedStillPictureHeader = uint8_t(br.read(1) ); + + bool decoderModelInfoPresent = false; + uint32_t bufferDelayLength = 0; + + if (_out.reducedStillPictureHeader) + { + _out.levelIdx = uint8_t(br.read(5) ); + } + else + { + if (0 != br.read(1) ) + { + br.read(32); + br.read(32); + + if (0 != br.read(1) ) + { + br.uvlc(); + } + + decoderModelInfoPresent = 0 != br.read(1); + + if (decoderModelInfoPresent) + { + bufferDelayLength = br.read(5) + 1; + br.read(32); + br.read(5); + br.read(5); + } + } + + const bool initialDisplayDelayPresent = 0 != br.read(1); + const uint32_t numOperatingPoints = br.read(5) + 1; + + for (uint32_t ii = 0; ii < numOperatingPoints; ++ii) + { + br.read(12); + const uint32_t levelIdx = br.read(5); + const uint32_t tier = 7 < levelIdx ? br.read(1) : 0; + + if (0 == ii) + { + _out.levelIdx = uint8_t(levelIdx); + _out.tier = uint8_t(tier); + } + + if (decoderModelInfoPresent + && 0 != br.read(1) ) + { + br.read(bufferDelayLength); + br.read(bufferDelayLength); + br.read(1); + } + + if (initialDisplayDelayPresent + && 0 != br.read(1) ) + { + br.read(4); + } + } + } + + const uint32_t widthBits = br.read(4) + 1; + const uint32_t heightBits = br.read(4) + 1; + _out.maxWidth = br.read(widthBits) + 1; + _out.maxHeight = br.read(heightBits) + 1; + + if (!_out.reducedStillPictureHeader + && 0 != br.read(1) ) + { + br.read(4); + br.read(3); + } + + br.read(3); + + if (!_out.reducedStillPictureHeader) + { + br.read(4); + const bool enableOrderHint = 0 != br.read(1); + + if (enableOrderHint) + { + br.read(2); + } + + const uint32_t forceScreenContentTools = 0 != br.read(1) ? 2 : br.read(1); + + if (0 < forceScreenContentTools + && 0 == br.read(1) ) + { + br.read(1); + } + + if (enableOrderHint) + { + br.read(3); + } + } + + br.read(3); + + _out.highBitdepth = uint8_t(br.read(1) ); + _out.twelveBit = av1::PROFILE_PROFESSIONAL == _out.profile && _out.highBitdepth ? uint8_t(br.read(1) ) : 0; + _out.monochrome = av1::PROFILE_HIGH == _out.profile ? 0 : uint8_t(br.read(1) ); + + uint32_t colorPrimaries = h273::CP_UNSPECIFIED; + uint32_t transfer = h273::TC_UNSPECIFIED; + uint32_t matrix = h273::MC_UNSPECIFIED; + + if (0 != br.read(1) ) + { + colorPrimaries = br.read(8); + transfer = br.read(8); + matrix = br.read(8); + } + + _out.matrixCoefficients = uint8_t(matrix); + + if (_out.monochrome) + { + _out.colorRange = uint8_t(br.read(1) ); + _out.subsamplingX = 1; + _out.subsamplingY = 1; + } + else if (h273::CP_BT709 == colorPrimaries + && h273::TC_SRGB == transfer + && h273::MC_IDENTITY == matrix) + { + _out.colorRange = 1; + } + else + { + _out.colorRange = uint8_t(br.read(1) ); + + if (av1::PROFILE_MAIN == _out.profile) + { + _out.subsamplingX = 1; + _out.subsamplingY = 1; + } + else if (av1::PROFILE_PROFESSIONAL == _out.profile) + { + if (_out.twelveBit) + { + _out.subsamplingX = uint8_t(br.read(1) ); + _out.subsamplingY = _out.subsamplingX ? uint8_t(br.read(1) ) : 0; + } + else + { + _out.subsamplingX = 1; + } + } + + if (_out.subsamplingX + && _out.subsamplingY) + { + _out.chromaSamplePosition = uint8_t(br.read(2) ); + } + } + + return !br.overrun() + && 0 != _out.maxWidth + && 0 != _out.maxHeight + ; + } + + template + static uint32_t visitObus(const uint8_t* _data, uint32_t _size, Fn _visit) + { + uint32_t count = 0; + uint32_t pos = 0; + + while (pos < _size) + { + const uint8_t header = _data[pos]; + const uint8_t type = (header >> av1::kObuTypeShift) & av1::kObuTypeMask; + const bool hasExtension = 0 != (header & av1::kObuExtensionFlag); + const bool hasSize = 0 != (header & av1::kObuHasSizeField); + + uint32_t payload = pos + 1 + (hasExtension ? 1 : 0); + + if (_size < payload) + { + break; + } + + uint32_t size = _size - payload; + + if (hasSize) + { + size = 0; + + for (uint32_t shift = 0; payload < _size && shift < 32; shift += 7) + { + const uint8_t byte = _data[payload++]; + size |= uint32_t(byte & av1::kLeb128ValueMask) << shift; + + if (0 == (byte & av1::kLeb128ContinuationBit) ) + { + break; + } + } + + size = bx::min(size, _size - payload); + } + + _visit(type, hasExtension ? _data[pos+1] : uint8_t(0), hasExtension, &_data[payload], size); + + count++; + pos = payload + size; + } + + return count; + } + + static uint32_t writeObuWithSize(uint8_t* _out, uint8_t _type, uint8_t _extension, bool _hasExtension, const uint8_t* _payload, uint32_t _payloadSize) + { + uint32_t outPos = 0; + + _out[outPos++] = uint8_t( (_type << av1::kObuTypeShift) | (_hasExtension ? av1::kObuExtensionFlag : 0) | av1::kObuHasSizeField); + + if (_hasExtension) + { + _out[outPos++] = _extension; + } + + uint32_t size = _payloadSize; + + do + { + uint8_t byte = uint8_t(size & av1::kLeb128ValueMask); + size >>= 7; + _out[outPos++] = uint8_t(byte | (0 != size ? av1::kLeb128ContinuationBit : 0) ); + } + while (0 != size); + + bx::memCopy(&_out[outPos], _payload, _payloadSize); + + return outPos + _payloadSize; + } + + static uint32_t av1TemporalUnitToSample(uint8_t* _out, const uint8_t* _data, uint32_t _size) + { + uint32_t outPos = 0; + + visitObus(_data, _size, [&](uint8_t _type, uint8_t _extension, bool _hasExtension, const uint8_t* _payload, uint32_t _payloadSize) + { + if (av1::OBU_TEMPORAL_DELIMITER != _type) + { + outPos += writeObuWithSize(&_out[outPos], _type, _extension, _hasExtension, _payload, _payloadSize); + } + }); + + return outPos; + } + + static bool peekAv1AuIsKeyFrame(const uint8_t* _data, uint32_t _size, bool _reducedStillPictureHeader) + { + bool seen = false; + bool key = false; + + visitObus(_data, _size, [&](uint8_t _type, uint8_t, bool, const uint8_t* _payload, uint32_t _payloadSize) + { + if (seen + || (av1::OBU_FRAME_HEADER != _type && av1::OBU_FRAME != _type) + || 0 == _payloadSize) + { + return; + } + + seen = true; + + if (_reducedStillPictureHeader) + { + key = true; + return; + } + + Av1BitReader br(_payload, _payloadSize); + + if (0 != br.read(1) ) + { + return; + } + + const uint32_t frameType = br.read(2); + const bool showFrame = 0 != br.read(1); + + key = av1::KEY_FRAME == frameType && showFrame; + }); + + return key; } struct VideoDecoderMtl { + struct ReorderedPicture + { + CVPixelBufferRef pb; + int32_t displayOrder; + int64_t ptsUs; + uint32_t generation; + }; + VideoDecoderMtl() : m_renderer(NULL) , m_device(NULL) , m_formatDesc(NULL) , m_session(NULL) , m_textureCache(NULL) - , m_queueHead(0) - , m_queueTail(0) + , m_codec(VideoCodec::Count) , m_dstWidth(0) , m_dstHeight(0) , m_initFlags(0) , m_cachedAuBytes(0) , m_dtsCounter(0) - , m_lastDisplayedPts(INT64_MIN) + , m_decodeCounter(0) + , m_displayedSlot(-1) + , m_prevDisplayedSlot(-1) + , m_sessionFailed(false) { - bx::memSet(m_queue, 0, sizeof(m_queue) ); + bx::memSet(&m_av1, 0, sizeof(m_av1) ); bx::memSet(m_yuvToRgb, 0, sizeof(m_yuvToRgb) ); + + for (uint32_t ii = 0; ii < kVideoMaxReorderInFlight; ++ii) + { + m_reorderPool[ii] = + { + .pb = NULL, + .displayOrder = -1, + .ptsUs = 0, + .generation = 0, + }; + } } bool create(const VideoDecoderInit& _init, RendererContextMtl* _renderer, MTL::Device* _device, uint16_t _width, uint16_t _height) { - m_renderer = _renderer; - m_device = _device; - m_dstWidth = _width; - m_dstHeight = _height; - m_initFlags = _init.flags; - m_cachedAuBytes = (0 != _init.cachedAuBytes) ? _init.cachedAuBytes : (4u << 20); - m_queueHead = 0; - m_queueTail = 0; - m_dtsCounter = 0; - m_lastDisplayedPts = INT64_MIN; + m_renderer = _renderer; + m_device = _device; + m_codec = _init.codec; + m_dstWidth = _width; + m_dstHeight = _height; + m_initFlags = _init.flags; + m_cachedAuBytes = (0 != _init.cachedAuBytes) ? _init.cachedAuBytes : (4u << 20); + m_auQueue.configure(m_cachedAuBytes, 0 != (m_initFlags & BGFX_VIDEO_DECODER_INIT_RETAIN) ); - bx::memSet(m_queue, 0, sizeof(m_queue) ); + bool created = false; - const uint8_t* psPtrs[16]; - size_t psSize[16]; - uint32_t psCount = 0; - const bool isHevc = (VideoCodec::H265 == _init.codec); - - visitAnnexBNalUnits(_init.parameterSets, _init.parameterSetsSize, [&](const uint8_t* _nal, uint32_t _nalSize) - { - if (0 == _nalSize - || psCount >= BX_COUNTOF(psPtrs) ) - { - return; - } - - bool keep = false; - - if (isHevc) - { - if (_nalSize < 2) return; - const uint8_t nalType = (_nal[0] >> 1) & 0x3F; - keep = (32 == nalType) - || (33 == nalType) - || (34 == nalType) - ; - } - else - { - const uint8_t nalType = _nal[0] & 0x1F; - keep = (7 == nalType) - || (8 == nalType) - ; - } - - if (keep) - { - psPtrs[psCount] = _nal; - psSize[psCount] = _nalSize; - ++psCount; - } - }); - - if (0 == psCount) + switch (m_codec) + { + case VideoCodec::H264: + case VideoCodec::H265: + created = createFormatDescriptionFromParameterSets(_init); + break; + + case VideoCodec::AV1: + created = createAv1FormatDescription(_init); + break; + + default: + break; + } + + if (!created) { - BX_TRACE("VideoDecoderMtl: no parameter set NAL units found (codec=%d).", int(_init.codec) ); return false; } - OSStatus status = noErr; - - if (isHevc) + if (VideoCodec::AV1 == m_codec) { - if (__builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, *) ) - { - status = CMVideoFormatDescriptionCreateFromHEVCParameterSets( - kCFAllocatorDefault - , psCount - , psPtrs - , psSize - , 4 - , NULL - , &m_formatDesc - ); - } - else - { - BX_TRACE("VideoDecoderMtl: HEVC requires macOS 10.13+."); - return false; - } + getYuvToRgb(m_yuvToRgb, m_av1.matrixCoefficients, false); } else { - status = CMVideoFormatDescriptionCreateFromH264ParameterSets( - kCFAllocatorDefault - , psCount - , psPtrs - , psSize - , 4 - , &m_formatDesc - ); + getYuvToRgb(m_yuvToRgb, getMatrixCoefficients(m_formatDesc), false); } - if (noErr != status - || NULL == m_formatDesc) - { - BX_TRACE("VideoDecoderMtl: CMVideoFormatDescriptionCreateFrom%sParameterSets failed: %d" - , isHevc ? "HEVC" : "H264" - , (int)status - ); - - return false; - } - - getYuvToRgb(m_yuvToRgb, getMatrixCoefficients(m_formatDesc), false); - if (!createSession() ) { CFRelease(m_formatDesc); @@ -277,23 +722,223 @@ namespace bgfx { namespace mtl return true; } + bool createFormatDescriptionFromParameterSets(const VideoDecoderInit& _init) + { + const uint8_t* psPtrs[kMaxParameterSets]; + size_t psSize[kMaxParameterSets]; + uint32_t psCount = 0; + const bool isHevc = VideoCodec::H265 == _init.codec; + + visitAnnexBNalUnits(_init.parameterSets, _init.parameterSetsSize, [&](const uint8_t* _nal, uint32_t _nalSize) + { + if (0 == _nalSize + || psCount >= BX_COUNTOF(psPtrs) ) + { + return; + } + + bool keep = false; + + if (isHevc) + { + if (_nalSize < hevc::kNalHeaderSize) + { + return; + } + + const hevc::NalUnitType nalType = hevc::getNalUnitType(_nal); + keep = hevc::VPS_NUT == nalType + || hevc::SPS_NUT == nalType + || hevc::PPS_NUT == nalType + ; + } + else + { + const uint8_t nalType = _nal[0] & kH264NalUnitTypeMask; + keep = h264::NAL_UNIT_TYPE_SPS == nalType + || h264::NAL_UNIT_TYPE_PPS == nalType + ; + } + + if (keep) + { + psPtrs[psCount] = _nal; + psSize[psCount] = _nalSize; + ++psCount; + } + }); + + if (0 == psCount) + { + BX_TRACE("VideoDecoderMtl: no parameter set NAL units found (codec=%d).", int(_init.codec) ); + return false; + } + + OSStatus status = noErr; + + if (isHevc) + { + if (__builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, *) ) + { + status = CMVideoFormatDescriptionCreateFromHEVCParameterSets( + kCFAllocatorDefault + , psCount + , psPtrs + , psSize + , kNalLengthSize + , NULL + , &m_formatDesc + ); + } + else + { + BX_TRACE("VideoDecoderMtl: HEVC requires macOS 10.13+."); + return false; + } + } + else + { + status = CMVideoFormatDescriptionCreateFromH264ParameterSets( + kCFAllocatorDefault + , psCount + , psPtrs + , psSize + , kNalLengthSize + , &m_formatDesc + ); + } + + if (noErr != status + || NULL == m_formatDesc) + { + BX_TRACE("VideoDecoderMtl: CMVideoFormatDescriptionCreateFrom%sParameterSets failed: %d" + , isHevc ? "HEVC" : "H264" + , (int)status + ); + + return false; + } + + return true; + } + + bool createAv1FormatDescription(const VideoDecoderInit& _init) + { +#if BGFX_VIDEO_METAL_AV1 + const uint8_t* seqPayload = NULL; + uint32_t seqPayloadSize = 0; + uint8_t seqExtension = 0; + bool seqHasExtension = false; + + visitObus(_init.parameterSets, _init.parameterSetsSize, [&](uint8_t _type, uint8_t _extension, bool _hasExtension, const uint8_t* _payload, uint32_t _payloadSize) + { + if (av1::OBU_SEQUENCE_HEADER == _type + && NULL == seqPayload) + { + seqPayload = _payload; + seqPayloadSize = _payloadSize; + seqExtension = _extension; + seqHasExtension = _hasExtension; + } + }); + + if (NULL == seqPayload + || !parseAv1SequenceHeader(seqPayload, seqPayloadSize, m_av1) ) + { + BX_TRACE("VideoDecoderMtl: no AV1 sequence header OBU found."); + return false; + } + + const uint8_t av1c[4] = + { + av1::kAv1CMarkerVersion, + uint8_t( (m_av1.profile << 5) | m_av1.levelIdx), + uint8_t( (m_av1.tier << 7) + | (m_av1.highBitdepth << 6) + | (m_av1.twelveBit << 5) + | (m_av1.monochrome << 4) + | (m_av1.subsamplingX << 3) + | (m_av1.subsamplingY << 2) + | m_av1.chromaSamplePosition + ), + 0, + }; + + uint8_t* seqObu = (uint8_t*)bx::alloc(g_allocator, seqPayloadSize + av1::kObuHeaderMaxSize); + const uint32_t seqObuSize = writeObuWithSize(seqObu, av1::OBU_SEQUENCE_HEADER, seqExtension, seqHasExtension, seqPayload, seqPayloadSize); + + CFMutableDataRef av1cData = CFDataCreateMutable(kCFAllocatorDefault, 0); + CFDataAppendBytes(av1cData, av1c, sizeof(av1c) ); + CFDataAppendBytes(av1cData, seqObu, seqObuSize); + bx::free(g_allocator, seqObu); + + const void* atomKeys[] = { CFSTR("av1C") }; + const void* atomValues[] = { av1cData }; + CFDictionaryRef atoms = CFDictionaryCreate( + kCFAllocatorDefault + , atomKeys + , atomValues + , 1 + , &kCFTypeDictionaryKeyCallBacks + , &kCFTypeDictionaryValueCallBacks + ); + CFRelease(av1cData); + + const void* extKeys[] = { kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms }; + const void* extValues[] = { atoms }; + CFDictionaryRef extensions = CFDictionaryCreate( + kCFAllocatorDefault + , extKeys + , extValues + , 1 + , &kCFTypeDictionaryKeyCallBacks + , &kCFTypeDictionaryValueCallBacks + ); + CFRelease(atoms); + + const OSStatus status = CMVideoFormatDescriptionCreate( + kCFAllocatorDefault + , kCMVideoCodecType_AV1 + , int32_t(m_av1.maxWidth) + , int32_t(m_av1.maxHeight) + , extensions + , &m_formatDesc + ); + CFRelease(extensions); + + if (noErr != status + || NULL == m_formatDesc) + { + BX_TRACE("VideoDecoderMtl: CMVideoFormatDescriptionCreate(AV1) failed: %d", (int)status); + return false; + } + + return true; +#else + BX_UNUSED(_init); + return false; +#endif // BGFX_VIDEO_METAL_AV1 + } + bool createSession() { const int32_t pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; - CFNumberRef pixelFormatNum = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat); + CFNumberRef pixelFormatNum = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat); + CFDictionaryRef ioSurfaceProps = CFDictionaryCreate(kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + const void* keys[] = { kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferMetalCompatibilityKey, kCVPixelBufferIOSurfacePropertiesKey }; - const void* values[] = { pixelFormatNum, kCFBooleanTrue, CFDictionaryCreate(kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks) }; + const void* values[] = { pixelFormatNum, kCFBooleanTrue, ioSurfaceProps }; CFDictionaryRef destAttrs = CFDictionaryCreate( kCFAllocatorDefault , keys , values - , 3 + , BX_COUNTOF(keys) , &kCFTypeDictionaryKeyCallBacks , &kCFTypeDictionaryValueCallBacks ); CFRelease(pixelFormatNum); - CFRelease(values[2]); + CFRelease(ioSurfaceProps); VTDecompressionOutputCallbackRecord callback; callback.decompressionOutputCallback = &VideoDecoderMtl::decodeOutputCallback; @@ -310,9 +955,11 @@ namespace bgfx { namespace mtl CFRelease(destAttrs); - if (noErr != status || NULL == m_session) + if (noErr != status + || NULL == m_session) { BX_TRACE("VideoDecoderMtl: VTDecompressionSessionCreate failed: %d", (int)status); + m_session = NULL; return false; } @@ -321,47 +968,98 @@ namespace bgfx { namespace mtl return true; } + void destroySession() + { + if (NULL != m_session) + { + VTDecompressionSessionWaitForAsynchronousFrames(m_session); + VTDecompressionSessionInvalidate(m_session); + CFRelease(m_session); + m_session = NULL; + } + } + + void releasePicture(uint32_t _poolIdx) + { + ReorderedPicture& pic = m_reorderPool[_poolIdx]; + + bx::MutexScope lock(m_poolMutex); + + if (NULL != pic.pb) + { + CFRelease(pic.pb); + } + + pic.pb = NULL; + pic.displayOrder = -1; + pic.ptsUs = 0; + ++pic.generation; + } + + CVPixelBufferRef acquirePicture(uint32_t _poolIdx) + { + bx::MutexScope lock(m_poolMutex); + + CVPixelBufferRef pb = m_reorderPool[_poolIdx].pb; + + if (NULL != pb) + { + CFRetain(pb); + } + + return pb; + } + + bool takeSessionFailed() + { + bx::MutexScope lock(m_poolMutex); + + const bool failed = m_sessionFailed; + m_sessionFailed = false; + + return failed; + } + + void waitForPendingFrames() + { + if (NULL != m_session) + { + VTDecompressionSessionWaitForAsynchronousFrames(m_session); + } + } + + void releasePictures() + { + for (uint32_t ii = 0; ii < kVideoMaxReorderInFlight; ++ii) + { + releasePicture(ii); + } + } + void resetForSet() { - if (NULL != m_session) + waitForPendingFrames(); + + if (m_auQueue.isRetaining() ) { - VTDecompressionSessionWaitForAsynchronousFrames(m_session); - VTDecompressionSessionInvalidate(m_session); - CFRelease(m_session); - m_session = NULL; + m_auQueue.rewindTo(m_auQueue.size() ); + } + else + { + m_auQueue.clear(); } - if (NULL != m_textureCache) - { - CVMetalTextureCacheFlush(m_textureCache, 0); - } + releasePictures(); - { - bx::MutexScope lock(m_queueMutex); - - while (m_queueHead != m_queueTail) - { - CFRelease(m_queue[m_queueHead % kQueueCapacity].pb); - ++m_queueHead; - } - - m_queueHead = 0; - m_queueTail = 0; - } - - m_dtsCounter = 0; - m_lastDisplayedPts = INT64_MIN; - createSession(); + m_displayedSlot = -1; + m_prevDisplayedSlot = -1; } + void destroy() { - if (NULL != m_session) - { - VTDecompressionSessionWaitForAsynchronousFrames(m_session); - VTDecompressionSessionInvalidate(m_session); - CFRelease(m_session); - m_session = NULL; - } + destroySession(); + releasePictures(); + m_auQueue.clear(); if (NULL != m_textureCache) { @@ -370,16 +1068,6 @@ namespace bgfx { namespace mtl m_textureCache = NULL; } - { - bx::MutexScope lock(m_queueMutex); - - while (m_queueHead != m_queueTail) - { - CFRelease(m_queue[m_queueHead % kQueueCapacity].pb); - ++m_queueHead; - } - } - if (NULL != m_formatDesc) { CFRelease(m_formatDesc); @@ -387,9 +1075,22 @@ namespace bgfx { namespace mtl } } + bool peekAuIsKeyFrame(const uint8_t* _data, uint32_t _size) const + { + switch (m_codec) + { + case VideoCodec::H264: return peekAuIsIdr(_data, _size); + case VideoCodec::H265: return peekHevcAuIsIrap(_data, _size); + case VideoCodec::AV1: return peekAv1AuIsKeyFrame(_data, _size, 0 != m_av1.reducedStillPictureHeader); + default: break; + } + + return false; + } + bool decode(const VideoDecoderFrame& _frame, MTL::Texture* _dst) { - if (NULL == m_session) + if (NULL == m_formatDesc) { return false; } @@ -397,183 +1098,224 @@ namespace bgfx { namespace mtl if (0 != (_frame.flags & BGFX_VIDEO_DECODE_FRAME_SET) ) { resetForSet(); + } - if (NULL == m_session) + if (NULL != _frame.bitstream + && 0 != _frame.numAus) + { + const uint8_t* bs = _frame.bitstream; + + for (uint32_t ii = 0; ii < _frame.numAus; ++ii) { - return false; + const uint32_t auSize = _frame.aus[ii].size; + m_auQueue.enqueue(bs, auSize, _frame.aus[ii].ptsUs, peekAuIsKeyFrame(bs, auSize) ); + bs += auSize; } } - const uint8_t* auBitstream = _frame.bitstream; - - for (uint32_t auIdx = 0; auIdx < _frame.numAus; ++auIdx) + while (!m_auQueue.empty() + && hasReorderSpace(m_reorderPool, kVideoMaxReorderInFlight) ) { - const uint32_t auSize = _frame.aus[auIdx].size; - const int64_t auPts = _frame.aus[auIdx].ptsUs; - - const uint32_t nalCount = visitAnnexBNalUnits(auBitstream, auSize, [](const uint8_t*, uint32_t){}); - const uint32_t avccBufSize = auSize + nalCount; - uint8_t* avccBytes = (uint8_t*)bx::alloc(g_allocator, avccBufSize); - const uint32_t avccSize = annexBToAvcc(avccBytes, auBitstream, auSize); - - BX_ASSERT(avccSize <= avccBufSize - , "annexBToAvcc wrote %u bytes into a %u-byte buffer (auSize=%u nalCount=%u)" - , avccSize, avccBufSize, auSize, nalCount - ); - - auBitstream += auSize; - - CMBlockBufferCustomBlockSource customSource = {}; - customSource.version = kCMBlockBufferCustomBlockSourceVersion; - customSource.FreeBlock = freeAvccBlockBxAlloc; - - CMBlockBufferRef blockBuffer = NULL; - OSStatus status = CMBlockBufferCreateWithMemoryBlock( - kCFAllocatorDefault - , avccBytes - , avccSize - , kCFAllocatorNull - , &customSource - , 0 - , avccSize - , 0 - , &blockBuffer - ); - - if (noErr != status) - { - bx::free(g_allocator, avccBytes); - - return false; - } - - const size_t sampleSize = avccSize; - CMSampleBufferRef sampleBuffer = NULL; - - const int64_t synthesizedDtsUs = int64_t(++m_dtsCounter * 1000ull); - - const CMSampleTimingInfo timing = - { - .duration = kCMTimeInvalid, - .presentationTimeStamp = CMTimeMake(auPts, 1000000), - .decodeTimeStamp = CMTimeMake(synthesizedDtsUs, 1000000), - }; - - status = CMSampleBufferCreate( - kCFAllocatorDefault - , blockBuffer - , true - , NULL - , NULL - , m_formatDesc - , 1 - , 1, &timing - , 1, &sampleSize - , &sampleBuffer - ); - - CFRelease(blockBuffer); - - if (noErr != status) - { - return false; - } - - VTDecodeFrameFlags flags = kVTDecodeFrame_EnableAsynchronousDecompression - | kVTDecodeFrame_EnableTemporalProcessing - ; - VTDecodeInfoFlags infoFlags = 0; - status = VTDecompressionSessionDecodeFrame(m_session, sampleBuffer, flags, NULL, &infoFlags); - - CFRelease(sampleBuffer); - - if (noErr != status) - { - BX_TRACE("VideoDecoderMtl: DecodeFrame submit failed: %d", (int)status); - - return false; - } + const PendingAU& au = m_auQueue.front(); + decodeOneAU(au.data.data(), uint32_t(au.data.size() ), au.ptsUs); + m_auQueue.pop(); } - const int64_t presentUs = _frame.presentationTimeUs; + m_auQueue.compact(); - if (0 == presentUs && 0 != _frame.numAus) + if (0 == _frame.numAus + && 0 == (_frame.flags & BGFX_VIDEO_DECODE_FRAME_NO_BLIT) ) { - return true; + int64_t adjustedClock = _frame.presentationTimeUs; + + if (0 != (_frame.flags & BGFX_VIDEO_DECODE_FRAME_LOOP) + && m_auQueue.isRetaining() + && m_auQueue.minPts() < m_auQueue.maxPts() ) + { + adjustedClock = applyLoopWrap(adjustedClock, m_auQueue.minPts(), m_auQueue.maxPts(), m_auQueue.size() ); + + const bool poolCovers = reorderPoolCovers(m_auQueue, m_reorderPool, kVideoMaxReorderInFlight, adjustedClock); + const bool forwardCanCatchUp = true + && !m_auQueue.empty() + && m_auQueue.front().ptsUs <= adjustedClock + 500000 + ; + + if (!poolCovers + && !forwardCanCatchUp) + { + faultInFromCache( + m_auQueue + , adjustedClock + , [this]() { resetForSet(); } + , [this](const uint8_t* _bs, uint32_t _bsSize, int64_t _pts) + { + decodeOneAU(_bs, _bsSize, _pts); + } + , m_reorderPool + , kVideoMaxReorderInFlight + ); + } + } + + bgfx::pickDisplaySlotForTime( + adjustedClock + , m_reorderPool + , kVideoMaxReorderInFlight + , m_displayedSlot + , m_prevDisplayedSlot + , m_auQueue.empty() + , [this, _dst](uint32_t _slot) + { + dispatchYuvToRgb(_slot, _dst); + }); } - CVPixelBufferRef pic = NULL; - int64_t picPts = INT64_MIN; + return true; + } + + bool decodeOneAU(const uint8_t* _bitstream, uint32_t _bitstreamSize, int64_t _ptsUs) + { + if (takeSessionFailed() ) { - bx::MutexScope lock(m_queueMutex); + destroySession(); + } - uint32_t bestIdx = UINT32_MAX; - int64_t bestPts = INT64_MIN; + if (NULL == m_session + && !createSession() ) + { + return false; + } - for (uint32_t ii = m_queueHead; ii != m_queueTail; ++ii) - { - const int64_t pts = m_queue[ii % kQueueCapacity].ptsUs; - if (pts <= presentUs && pts > bestPts) - { - bestPts = pts; - bestIdx = ii; - } - } + uint8_t* sampleBytes = NULL; + uint32_t sampleSize = 0; - if (UINT32_MAX == bestIdx - && m_queueHead != m_queueTail - && INT64_MIN == m_lastDisplayedPts) - { - bestIdx = m_queueHead; - bestPts = m_queue[bestIdx % kQueueCapacity].ptsUs; + if (VideoCodec::AV1 == m_codec) + { + const uint32_t numObus = visitObus(_bitstream, _bitstreamSize, [](uint8_t, uint8_t, bool, const uint8_t*, uint32_t) {}); + sampleBytes = (uint8_t*)bx::alloc(g_allocator, _bitstreamSize + av1::kObuHeaderMaxSize*numObus); + sampleSize = av1TemporalUnitToSample(sampleBytes, _bitstream, _bitstreamSize); + } + else + { + const uint32_t numNals = visitAnnexBNalUnits(_bitstream, _bitstreamSize, [](const uint8_t*, uint32_t) {}); + sampleBytes = (uint8_t*)bx::alloc(g_allocator, _bitstreamSize + kNalLengthSize*numNals); + sampleSize = annexBToLengthPrefixed(sampleBytes, _bitstream, _bitstreamSize); + } - for (uint32_t ii = m_queueHead + 1; ii != m_queueTail; ++ii) - { - const int64_t pts = m_queue[ii % kQueueCapacity].ptsUs; + if (0 == sampleSize) + { + bx::free(g_allocator, sampleBytes); + return false; + } - if (pts < bestPts) - { - bestPts = pts; - bestIdx = ii; - } - } - } + CMBlockBufferCustomBlockSource customSource = {}; + customSource.version = kCMBlockBufferCustomBlockSourceVersion; + customSource.FreeBlock = freeBlockBxAlloc; - if (UINT32_MAX != bestIdx) - { - pic = m_queue[bestIdx % kQueueCapacity].pb; - picPts = bestPts; - CFRetain(pic); + CMBlockBufferRef blockBuffer = NULL; + OSStatus status = CMBlockBufferCreateWithMemoryBlock( + kCFAllocatorDefault + , sampleBytes + , sampleSize + , kCFAllocatorNull + , &customSource + , 0 + , sampleSize + , 0 + , &blockBuffer + ); - uint32_t writeIdx = m_queueHead; + if (noErr != status) + { + bx::free(g_allocator, sampleBytes); + return false; + } - for (uint32_t ii = m_queueHead; ii != m_queueTail; ++ii) - { - QueueEntry& e = m_queue[ii % kQueueCapacity]; + const size_t sampleSizes[] = { sampleSize }; + const int64_t dtsUs = int64_t(++m_dtsCounter * 1000ull); - if (e.ptsUs <= picPts) - { - CFRelease(e.pb); - } - else - { - m_queue[writeIdx % kQueueCapacity] = e; - ++writeIdx; - } - } + const CMSampleTimingInfo timing = + { + .duration = kCMTimeInvalid, + .presentationTimeStamp = CMTimeMake(_ptsUs, 1000000), + .decodeTimeStamp = CMTimeMake(dtsUs, 1000000), + }; - m_queueTail = writeIdx; - m_lastDisplayedPts = picPts; - } + CMSampleBufferRef sampleBuffer = NULL; + status = CMSampleBufferCreate( + kCFAllocatorDefault + , blockBuffer + , true + , NULL + , NULL + , m_formatDesc + , 1 + , 1, &timing + , 1, sampleSizes + , &sampleBuffer + ); + + CFRelease(blockBuffer); + + if (noErr != status) + { + return false; + } + + const uint32_t poolIdx = pickReorderSlotOrEvictOldest(m_reorderPool, kVideoMaxReorderInFlight); + releasePicture(poolIdx); + + ReorderedPicture& pic = m_reorderPool[poolIdx]; + pic.displayOrder = int32_t(m_decodeCounter++ & 0x7fffffff); + pic.ptsUs = _ptsUs; + + const uintptr_t refCon = (uintptr_t(pic.generation) << 32) | uintptr_t(poolIdx); + + VTDecodeInfoFlags infoFlags = 0; + status = VTDecompressionSessionDecodeFrame( + m_session + , sampleBuffer + , kVTDecodeFrame_EnableAsynchronousDecompression + , (void*)refCon + , &infoFlags + ); + + CFRelease(sampleBuffer); + + if (noErr != status) + { + BX_TRACE("VideoDecoderMtl: VTDecompressionSessionDecodeFrame failed: %d", (int)status); + releasePicture(poolIdx); + destroySession(); + return false; + } + + return true; + } + + void dispatchYuvToRgb(uint32_t _poolIdx, MTL::Texture* _dst) + { + BX_ASSERT(NULL != g_videoDecode, "Video decode program not initialized."); + BX_ASSERT(_poolIdx < kVideoMaxReorderInFlight, "Reorder pool index out of range."); + + if (NULL == _dst) + { + return; + } + + CVPixelBufferRef pic = acquirePicture(_poolIdx); + + if (NULL == pic) + { + waitForPendingFrames(); + pic = acquirePicture(_poolIdx); } if (NULL == pic) { - return true; + return; } - BX_UNUSED(picPts); - const size_t yWidth = CVPixelBufferGetWidthOfPlane (pic, 0); const size_t yHeight = CVPixelBufferGetHeightOfPlane(pic, 0); const size_t cWidth = CVPixelBufferGetWidthOfPlane (pic, 1); @@ -597,7 +1339,7 @@ namespace bgfx { namespace mtl { BX_TRACE("VideoDecoderMtl: Y plane texture create failed: %d", (int)cvr); CFRelease(pic); - return false; + return; } cvr = CVMetalTextureCacheCreateTextureFromImage( @@ -616,23 +1358,27 @@ namespace bgfx { namespace mtl BX_TRACE("VideoDecoderMtl: CbCr plane texture create failed: %d", (int)cvr); CFRelease(yMtlTex); CFRelease(pic); - return false; + return; } - MTL::Texture* yTex = (MTL::Texture*)CVMetalTextureGetTexture(yMtlTex); - MTL::Texture* cTex = (MTL::Texture*)CVMetalTextureGetTexture(cMtlTex); + const TextureFormat::Enum dstFormat = MTL::PixelFormatRGBA16Float == _dst->pixelFormat() + ? TextureFormat::RGBA16F + : TextureFormat::RGBA8 + ; - BX_ASSERT(NULL != g_videoDecode, "g_videoDecode not initialized."); - PipelineStateMtl* pso = videoGetComputePipelineState(m_renderer, g_videoDecode->m_program[VideoDecode::Unorm]); + PipelineStateMtl* pso = videoGetComputePipelineState(m_renderer, g_videoDecode->getProgram(dstFormat) ); if (NULL == pso) { CFRelease(yMtlTex); CFRelease(cMtlTex); CFRelease(pic); - return false; + return; } + MTL::Texture* yTex = (MTL::Texture*)CVMetalTextureGetTexture(yMtlTex); + MTL::Texture* cTex = (MTL::Texture*)CVMetalTextureGetTexture(cMtlTex); + videoEndEncoding(m_renderer); MTL::CommandBuffer* cmdBuf = videoEnsureCommandBuffer(m_renderer); @@ -640,8 +1386,8 @@ namespace bgfx { namespace mtl MTL::ComputeCommandEncoder* cce = cmdBuf->computeCommandEncoder(); cce->setComputePipelineState(pso->m_cps); - MTL::SamplerState* linearClamp = videoGetSamplerState(m_renderer, - BGFX_SAMPLER_U_CLAMP + MTL::SamplerState* linearClamp = videoGetSamplerState(m_renderer, 0 + | BGFX_SAMPLER_U_CLAMP | BGFX_SAMPLER_V_CLAMP | BGFX_SAMPLER_W_CLAMP ); @@ -662,118 +1408,83 @@ namespace bgfx { namespace mtl cce->dispatchThreadgroups(groupCount, threadsPerGroup); cce->endEncoding(); - CVPixelBufferRef retainedPic = pic; - CVMetalTextureRef retainedY = yMtlTex; - CVMetalTextureRef retainedC = cMtlTex; - - MTL::HandlerFunction releaseRefs = [retainedPic, retainedY, retainedC](MTL::CommandBuffer*) + MTL::HandlerFunction releaseRefs = [pic, yMtlTex, cMtlTex](MTL::CommandBuffer*) { - CFRelease(retainedY); - CFRelease(retainedC); - CFRelease(retainedPic); + CFRelease(yMtlTex); + CFRelease(cMtlTex); + CFRelease(pic); }; cmdBuf->addCompletedHandler(releaseRefs); - - return true; } static void decodeOutputCallback( void* _refCon - , void* /*_sourceFrameRefCon*/ + , void* _sourceFrameRefCon , OSStatus _status - , VTDecodeInfoFlags /*_infoFlags*/ + , VTDecodeInfoFlags _infoFlags , CVImageBufferRef _imageBuffer - , CMTime _pts + , CMTime /*_pts*/ , CMTime /*_duration*/ ) { - VideoDecoderMtl* self = (VideoDecoderMtl*)_refCon; + VideoDecoderMtl* self = (VideoDecoderMtl*)_refCon; + const uintptr_t refCon = uintptr_t(_sourceFrameRefCon); + const uint32_t poolIdx = uint32_t(refCon & UINT32_MAX); + const uint32_t generation = uint32_t(refCon >> 32); - if (noErr != _status - || NULL == _imageBuffer) + bx::MutexScope lock(self->m_poolMutex); + + if (noErr != _status) + { + self->m_sessionFailed = true; + return; + } + + if (NULL == _imageBuffer + || 0 != (_infoFlags & kVTDecodeInfo_FrameDropped) + || kVideoMaxReorderInFlight <= poolIdx) { return; } - const int64_t ptsUs = CMTIME_IS_VALID(_pts) && _pts.timescale > 0 - ? int64_t(double(_pts.value) * 1000000.0 / double(_pts.timescale) ) - : INT64_MIN; + ReorderedPicture& pic = self->m_reorderPool[poolIdx]; - bx::MutexScope lock(self->m_queueMutex); - - if (self->m_queueTail - self->m_queueHead >= kQueueCapacity) + if (generation != pic.generation) { - const int64_t newPts = ptsUs; - uint32_t worstIdx = self->m_queueHead; - int64_t worstPts = self->m_queue[worstIdx % kQueueCapacity].ptsUs; - - for (uint32_t ii = self->m_queueHead + 1; ii != self->m_queueTail; ++ii) - { - const int64_t pts = self->m_queue[ii % kQueueCapacity].ptsUs; - - if (pts > worstPts) - { - worstPts = pts; - worstIdx = ii; - } - } - - if (newPts >= worstPts) - { - return; - } - - CFRelease(self->m_queue[worstIdx % kQueueCapacity].pb); - uint32_t writeIdx = self->m_queueHead; - - for (uint32_t ii = self->m_queueHead; ii != self->m_queueTail; ++ii) - { - if (ii == worstIdx) - { - continue; - } - - self->m_queue[writeIdx % kQueueCapacity] = self->m_queue[ii % kQueueCapacity]; - ++writeIdx; - } - - self->m_queueTail = writeIdx; + return; } - CFRetain(_imageBuffer); - QueueEntry& e = self->m_queue[self->m_queueTail % kQueueCapacity]; - e.pb = (CVPixelBufferRef)_imageBuffer; - e.ptsUs = ptsUs; - ++self->m_queueTail; + if (NULL != pic.pb) + { + CFRelease(pic.pb); + } + + pic.pb = (CVPixelBufferRef)CFRetain(_imageBuffer); } - static constexpr uint32_t kQueueCapacity = 192; - static constexpr uint32_t kReorderDepth = 4; - - struct QueueEntry - { - CVPixelBufferRef pb; - int64_t ptsUs; - }; - RendererContextMtl* m_renderer; MTL::Device* m_device; CMVideoFormatDescriptionRef m_formatDesc; VTDecompressionSessionRef m_session; CVMetalTextureCacheRef m_textureCache; - bx::Mutex m_queueMutex; - QueueEntry m_queue[kQueueCapacity]; - uint32_t m_queueHead; - uint32_t m_queueTail; - uint16_t m_dstWidth; - uint16_t m_dstHeight; - uint32_t m_initFlags; - uint32_t m_cachedAuBytes; - uint64_t m_dtsCounter; - int64_t m_lastDisplayedPts; - float m_yuvToRgb[12]; + + bx::Mutex m_poolMutex; + AuQueue m_auQueue; + ReorderedPicture m_reorderPool[kVideoMaxReorderInFlight]; + Av1SequenceHeader m_av1; + VideoCodec::Enum m_codec; + uint16_t m_dstWidth; + uint16_t m_dstHeight; + uint32_t m_initFlags; + uint32_t m_cachedAuBytes; + uint64_t m_dtsCounter; + uint32_t m_decodeCounter; + int32_t m_displayedSlot; + int32_t m_prevDisplayedSlot; + bool m_sessionFailed; + float m_yuvToRgb[12]; }; VideoDecoderMtl* videoDecoderCreate(const VideoDecoderInit& _init, RendererContextMtl* _renderer, MTL::Device* _device, uint16_t _width, uint16_t _height) @@ -817,9 +1528,9 @@ namespace bgfx { namespace mtl { { VideoCodec::H264, kCMVideoCodecType_H264, BGFX_CAPS_VIDEO_CODEC_BIT_8, BGFX_CAPS_VIDEO_CODEC_CHROMA_420 }, { VideoCodec::H265, kCMVideoCodecType_HEVC, BGFX_CAPS_VIDEO_CODEC_BIT_8 | BGFX_CAPS_VIDEO_CODEC_BIT_10, BGFX_CAPS_VIDEO_CODEC_CHROMA_420 }, -#if defined(__MAC_14_0) || defined(__IPHONE_17_0) +#if BGFX_VIDEO_METAL_AV1 { VideoCodec::AV1, kCMVideoCodecType_AV1, BGFX_CAPS_VIDEO_CODEC_BIT_8 | BGFX_CAPS_VIDEO_CODEC_BIT_10, BGFX_CAPS_VIDEO_CODEC_CHROMA_420 }, -#endif // defined(__MAC_14_0) || defined(__IPHONE_17_0) +#endif // BGFX_VIDEO_METAL_AV1 }; void initVideoDecoder() diff --git a/tools/texturev/texturev.cpp b/tools/texturev/texturev.cpp index 00040a660..923bb4d39 100644 --- a/tools/texturev/texturev.cpp +++ b/tools/texturev/texturev.cpp @@ -71,7 +71,7 @@ static const bgfx::EmbeddedShader s_embeddedShaders[] = BGFX_EMBEDDED_SHADER_END() }; -static bx::StringView s_supportedExt[32]; +static bx::StringView s_supportedExt[64]; static uint32_t s_numSupportedExt = 0; static int32_t compareExt(const void* _lhs, const void* _rhs) @@ -850,29 +850,55 @@ struct View && bx::FileType::File == fi.type) { bx::StringView ext = fi.filePath.getExt(); + bool supported = false; if (!ext.isEmpty() ) { ext.set(ext.getPtr()+1, ext.getTerm() ); + supported = isExtSupported(ext); + } + else + { + bx::FilePath fullPath = m_path; + fullPath.join(fi.filePath.getFileName() ); + supported = VideoPlayer::isVideoFile(fullPath); + } - if (isExtSupported(ext) ) - { - const bx::StringView fileName = fi.filePath.getFileName(); - m_fileList.push_back(std::string(fileName.getPtr(), fileName.getTerm() ) ); - } + if (supported) + { + const bx::StringView fileName = fi.filePath.getFileName(); + m_fileList.push_back(std::string(fileName.getPtr(), fileName.getTerm() ) ); } } } bx::close(&dr); + const bx::StringView fileName = _filePath.getFileName(); + + { + bx::FileInfo fi; + bool listed = false; + + for (FileList::const_iterator it = m_fileList.begin(); it != m_fileList.end(); ++it) + { + listed |= 0 == bx::strCmp(it->c_str(), fileName); + } + + if (!listed + && !fileName.isEmpty() + && bx::stat(fi, _filePath) + && bx::FileType::File == fi.type) + { + m_fileList.push_back(std::string(fileName.getPtr(), fileName.getTerm() ) ); + } + } + std::sort(m_fileList.begin(), m_fileList.end(), sortNameAscending); m_fileIndex = 0; uint32_t idx = 0; - const bx::StringView fileName = _filePath.getFileName(); - for (FileList::const_iterator it = m_fileList.begin(); it != m_fileList.end(); ++it, ++idx) { if (0 == bx::strCmpI(it->c_str(), fileName) ) @@ -1564,7 +1590,41 @@ int _main_(int _argc, char** _argv) if (0 != (caps->supported & BGFX_CAPS_VIDEO_DECODE) ) { - s_supportedExt[s_numSupportedExt++] = "mp4"; + struct CodecExt + { + bgfx::VideoCodec::Enum codec; + const char* const* ext; + }; + + static const char* const s_containerExt[] = { "mp4", "m4v", "mov", NULL }; + static const char* const s_h264Ext[] = { "264", "h264", NULL }; + static const char* const s_h265Ext[] = { "265", "h265", "hevc", NULL }; + static const char* const s_av1Ext[] = { "av1", "obu", "ivf", NULL }; + + static const CodecExt s_codecExt[] = + { + { bgfx::VideoCodec::Count, s_containerExt }, + { bgfx::VideoCodec::H264, s_h264Ext }, + { bgfx::VideoCodec::H265, s_h265Ext }, + { bgfx::VideoCodec::AV1, s_av1Ext }, + }; + + for (uint32_t ii = 0; ii < BX_COUNTOF(s_codecExt); ++ii) + { + const CodecExt& codecExt = s_codecExt[ii]; + + if (bgfx::VideoCodec::Count != codecExt.codec + && 0 == caps->codecs[codecExt.codec]) + { + continue; + } + + for (const char* const* ext = codecExt.ext; NULL != *ext && s_numSupportedExt < BX_COUNTOF(s_supportedExt); ++ext) + { + s_supportedExt[s_numSupportedExt++] = *ext; + } + } + bx::quickSort(s_supportedExt, s_numSupportedExt, sizeof(s_supportedExt[0]), compareExt); } @@ -2453,18 +2513,7 @@ int _main_(int _argc, char** _argv) bx::FilePath fp = view.m_path; fp.join(view.m_fileList[view.m_fileIndex].c_str() ); - bx::StringView fileName(view.m_fileList[view.m_fileIndex].c_str() ); - - bx::StringView ext; - if (fileName.getLength() >= 4) - { - ext.set(fileName.getTerm() - 4, fileName.getTerm() ); - } - - const bool nextIsVideo = false - || 0 == bx::strCmpI(ext, ".mp4") - || 0 == bx::strCmpI(ext, ".m4v") - ; + const bool nextIsVideo = VideoPlayer::isVideoFile(fp); if (videoOwnsTexture) { diff --git a/tools/texturev/video_player.cpp b/tools/texturev/video_player.cpp index d25effbb9..daf6b3e0b 100644 --- a/tools/texturev/video_player.cpp +++ b/tools/texturev/video_player.cpp @@ -7,10 +7,13 @@ #include #include -#include -#include #include +#include #include +#include +#include +#include +#include #include @@ -28,32 +31,48 @@ BX_PRAGMA_DIAGNOSTIC_POP(); struct VideoPlayer::Impl { - bgfx::VideoCodec::Enum codec = bgfx::VideoCodec::Count; - uint8_t chroma = 1; + enum Container + { + Mp4, + AnnexB, + Obu, + Ivf, + }; + + Container container = Mp4; + bgfx::VideoCodec::Enum codec = bgfx::VideoCodec::Count; + uint8_t chroma = 0; uint8_t bitDepth = 8; uint8_t profile = 0; uint8_t level = 0; uint16_t width = 0; uint16_t height = 0; - uint8_t maxDpbSlots = 0; - uint8_t maxActiveReferences = 0; + uint8_t maxDpbSlots = 16; + uint8_t maxActiveReferences = 4; stl::vector parameterSets; - lsmash_root_t* root = NULL; - lsmash_file_parameters_t fp = {}; - bool fpOpened = false; - uint32_t trackId = 0; - uint32_t lengthSize = 4; + lsmash_root_t* root = NULL; + lsmash_file_parameters_t fp = {}; + bool fpOpened = false; + uint32_t trackId = 0; + uint32_t lengthSize = 4; - stl::vector annexBScratch; + bx::FileReader reader; + bool readerOpened = false; + uint64_t fileSize = 0; + + stl::vector scratch; struct Frame { uint64_t ptsUs; + uint64_t offset; + uint32_t size; bool isKeyframe; }; stl::vector frames; + uint64_t totalBytes = 0; ~Impl() { @@ -68,17 +87,147 @@ struct VideoPlayer::Impl lsmash_close_file(&fp); fpOpened = false; } + + if (readerOpened) + { + bx::close(&reader); + readerOpened = false; + } } }; namespace { + typedef VideoPlayer::Impl Impl; + const uint32_t kMaxSubmitPerCall = 1; const int64_t kLookaheadUs = 2 * 1000000; - const int64_t kSkipStepUs = 8 * 1000000; + const int64_t kCacheWindowUs = 2 * kLookaheadUs; + const uint32_t kMinCachedAuBytes = 16u << 20; const float kRateStepFactor = 1.25892541f; const float kRateMin = 0.1f; const float kRateMax = 100.0f; + const int64_t kDefaultFrameUs = 1000000 / 30; + const uint32_t kProbeSize = 64; + const uint32_t kNalPeekSize = 1024; + const uint32_t kScanBufferSize = 1 << 20; + const uint32_t kBoxHeaderSize = 8; + const uint8_t kH264NalUnitTypeMask = 0x1f; + + namespace hevc + { + enum NalUnitType + { + BLA_W_LP = 16, + BLA_N_LP = 18, + IDR_W_RADL = 19, + IDR_N_LP = 20, + CRA_NUT = 21, + RSV_IRAP_VCL23 = 23, + VPS_NUT = 32, + SPS_NUT = 33, + PPS_NUT = 34, + AUD_NUT = 35, + PREFIX_SEI_NUT = 39, + SUFFIX_SEI_NUT = 40, + }; + + const uint32_t kNalHeaderSize = 2; + const uint32_t kMaxSpsCount = 16; + const uint32_t kMaxPpsCount = 64; + const uint32_t kMaxSubLayers = 7; + const uint32_t kMaxStRefPicSets = 64; + + inline uint8_t getNalUnitType(const uint8_t* _nal) + { + return (_nal[0] >> 1) & 0x3f; + } + + inline bool isVcl(uint8_t _type) + { + return bx::isInRange(_type, 0, CRA_NUT); + } + + inline bool isIrap(uint8_t _type) + { + return bx::isInRange(_type, BLA_W_LP, RSV_IRAP_VCL23); + } + + inline bool isIdr(uint8_t _type) + { + return IDR_W_RADL == _type || IDR_N_LP == _type; + } + + inline bool isIdrOrBla(uint8_t _type) + { + return bx::isInRange(_type, BLA_W_LP, IDR_N_LP); + } + + inline bool isParameterSet(uint8_t _type) + { + return bx::isInRange(_type, VPS_NUT, PPS_NUT); + } + } + + namespace av1 + { + enum ObuType + { + OBU_SEQUENCE_HEADER = 1, + OBU_TEMPORAL_DELIMITER = 2, + OBU_FRAME_HEADER = 3, + OBU_FRAME = 6, + }; + + enum FrameType + { + KEY_FRAME = 0, + }; + + enum Profile + { + PROFILE_MAIN = 0, + PROFILE_HIGH = 1, + PROFILE_PROFESSIONAL = 2, + }; + + const uint32_t kObuTypeShift = 3; + const uint8_t kObuTypeMask = 0xf; + const uint8_t kObuExtensionFlag = 0x4; + const uint8_t kObuHasSizeField = 0x2; + const uint32_t kObuHeaderMaxSize = 16; + const uint32_t kMaxLeadingZeros = 32; + const uint8_t kLeb128ValueMask = 0x7f; + const uint8_t kLeb128ContinuationBit = 0x80; + const uint32_t kAv1CSize = 4; + } + + namespace ivf + { + const uint32_t kHeaderSize = 32; + const uint32_t kFrameHeaderSize = 12; + } + + template + Ty readLe(const uint8_t* _data) + { + Ty value; + bx::memCopy(&value, _data, sizeof(value) ); + return bx::toHostEndian(value, true); + } + + template + Ty readBe(const uint8_t* _data) + { + Ty value; + bx::memCopy(&value, _data, sizeof(value) ); + return bx::toHostEndian(value, false); + } + + bool isFourCC(const uint8_t* _data, const char* _fourcc) + { + return 0 == bx::memCmp(_data, _fourcc, 4); + } void avccToAnnexB(stl::vector& _out, const uint8_t* _in, uint32_t _size, uint32_t _lengthSize) { @@ -109,6 +258,32 @@ namespace } } + template + void visitLengthPrefixedNals(const uint8_t* _data, uint32_t _size, uint32_t _lengthSize, Fn _visit) + { + uint32_t pos = 0; + + while (pos + _lengthSize <= _size) + { + uint32_t naluLen = 0; + + for (uint32_t ii = 0; ii < _lengthSize; ++ii) + { + naluLen = (naluLen << 8) | _data[pos + ii]; + } + + pos += _lengthSize; + + if (pos + naluLen > _size) + { + break; + } + + _visit(_data + pos, naluLen); + pos += naluLen; + } + } + uint32_t appendNalAsAnnexB(stl::vector& _out, const uint8_t* _nal, uint32_t _nalLen) { if (NULL == _nal @@ -125,12 +300,10 @@ namespace return 1; } - void stripEmulationPrevention(stl::vector& _dst, const uint8_t* _src, uint32_t _size) + uint32_t stripEmulationPrevention(uint8_t* _dst, const uint8_t* _src, uint32_t _size) { - _dst.clear(); - _dst.reserve(_size); - - uint32_t zeros = 0; + uint32_t dstIdx = 0; + uint32_t zeros = 0; for (uint32_t ii = 0; ii < _size; ++ii) { @@ -142,14 +315,1737 @@ namespace } zeros = (0 == _src[ii]) ? zeros + 1 : 0; - _dst.push_back(_src[ii]); + _dst[dstIdx++] = _src[ii]; + } + + return dstIdx; + } + + void stripEmulationPrevention(stl::vector& _dst, const uint8_t* _src, uint32_t _size) + { + _dst.resize(_size); + _dst.resize(stripEmulationPrevention(_dst.data(), _src, _size) ); + } + + uint8_t chromaFromFormatIdc(uint32_t _chromaFormatIdc) + { + return 3 == _chromaFormatIdc ? 4 + : 2 == _chromaFormatIdc ? 2 + : 0 + ; + } + + int32_t computePoc( + int32_t& _prevPocMsb + , int32_t& _prevPocLsb + , const h264::SPS& _sps + , const h264::SliceHeader& _sh + , const h264::NALHeader& _nal + , bool _isIdr + ) + { + if (2 == _sps.pic_order_cnt_type) + { + const int32_t maxFrameNum = 1 << (_sps.log2_max_frame_num_minus4 + 4); + int32_t frameNumOffset; + int32_t poc; + + if (_isIdr) + { + frameNumOffset = 0; + poc = 0; + } + else + { + frameNumOffset = _prevPocLsb > _sh.frame_num + ? _prevPocMsb + maxFrameNum + : _prevPocMsb + ; + + poc = 2 * (frameNumOffset + _sh.frame_num); + + if (0 == _nal.idc) + { + poc -= 1; + } + } + + _prevPocMsb = frameNumOffset; + _prevPocLsb = _sh.frame_num; + + return poc; + } + + if (0 != _sps.pic_order_cnt_type) + { + return _sh.frame_num * 2; + } + + const int32_t maxPocLsb = 1 << (_sps.log2_max_pic_order_cnt_lsb_minus4 + 4); + int32_t prevPocMsb = _isIdr ? 0 : _prevPocMsb; + int32_t prevPocLsb = _isIdr ? 0 : _prevPocLsb; + + const int32_t pocLsb = _sh.pic_order_cnt_lsb; + int32_t pocMsb; + + if (pocLsb < prevPocLsb + && prevPocLsb - pocLsb >= maxPocLsb / 2) + { + pocMsb = prevPocMsb + maxPocLsb; + } + else if (pocLsb > prevPocLsb + && pocLsb - prevPocLsb > maxPocLsb / 2) + { + pocMsb = prevPocMsb - maxPocLsb; + } + else + { + pocMsb = prevPocMsb; + } + + if (0 != _nal.idc) + { + _prevPocMsb = pocMsb; + _prevPocLsb = pocLsb; + } + + return pocMsb + pocLsb; + } + + struct H264Tables + { + H264Tables() + : prevPocMsb(0) + , prevPocLsb(0) + { + sps.resize(h264::MAX_SPS_ID + 1); + pps.resize(h264::MAX_PPS_ID + 1); + bx::memSet(spsValid, 0, sizeof(spsValid) ); + bx::memSet(ppsValid, 0, sizeof(ppsValid) ); + } + + stl::vector sps; + stl::vector pps; + bool spsValid[h264::MAX_SPS_ID + 1]; + bool ppsValid[h264::MAX_PPS_ID + 1]; + int32_t prevPocMsb; + int32_t prevPocLsb; + }; + + void noteH264Sps(Impl& _video, const h264::SPS& _sps, int64_t& _frameUs) + { + _video.profile = uint8_t(_sps.profile_idc); + _video.level = uint8_t(_sps.level_idc); + _video.chroma = chromaFromFormatIdc(_sps.chroma_format_idc); + _video.bitDepth = uint8_t(_sps.bit_depth_luma_minus8 + 8); + + const uint32_t frameMbsOnly = 0 != _sps.frame_mbs_only_flag ? 1 : 0; + const uint32_t codedWidth = uint32_t(_sps.pic_width_in_mbs_minus1 + 1) * 16; + const uint32_t codedHeight = uint32_t(_sps.pic_height_in_map_units_minus1 + 1) * 16 * (2 - frameMbsOnly); + + uint32_t width = codedWidth; + uint32_t height = codedHeight; + + if (0 != _sps.frame_cropping_flag) + { + const uint32_t cropUnitX = 2; + const uint32_t cropUnitY = 2 * (2 - frameMbsOnly); + const uint32_t cropX = uint32_t(_sps.frame_crop_left_offset + _sps.frame_crop_right_offset) * cropUnitX; + const uint32_t cropY = uint32_t(_sps.frame_crop_top_offset + _sps.frame_crop_bottom_offset) * cropUnitY; + + if (cropX < codedWidth + && cropY < codedHeight) + { + width = codedWidth - cropX; + height = codedHeight - cropY; + } + } + + _video.width = uint16_t(width); + _video.height = uint16_t(height); + + if (0 != _sps.vui_parameters_present_flag + && 0 != _sps.vui.timing_info_present_flag + && 0 < _sps.vui.time_scale + && 0 < _sps.vui.num_units_in_tick) + { + _frameUs = int64_t(2000000ull * uint64_t(_sps.vui.num_units_in_tick) / uint64_t(_sps.vui.time_scale) ); } } - bool parseAvcC(VideoPlayer::Impl& _video, const uint8_t* _data, uint32_t _size, uint32_t& _lengthSize) + bool parseH264ParameterSet(H264Tables& _tables, Impl& _video, int64_t& _frameUs, const uint8_t* _rbsp, uint32_t _size) { - const uint32_t kBoxHeaderSize = 8; + h264::Bitstream bs; + bs.init(_rbsp, _size); + h264::NALHeader nal = {}; + + if (!h264::read_nal_header(&nal, &bs) ) + { + return false; + } + + if (h264::NAL_UNIT_TYPE_SPS == nal.type) + { + h264::SPS sps = {}; + h264::read_sps(&sps, &bs); + + if (sps.seq_parameter_set_id < 0 + || sps.seq_parameter_set_id > h264::MAX_SPS_ID) + { + return false; + } + + _tables.sps[sps.seq_parameter_set_id] = sps; + _tables.spsValid[sps.seq_parameter_set_id] = true; + noteH264Sps(_video, sps, _frameUs); + + return true; + } + + if (h264::NAL_UNIT_TYPE_PPS == nal.type) + { + h264::PPS pps = {}; + h264::read_pps(&pps, &bs); + + if (pps.pic_parameter_set_id < 0 + || pps.pic_parameter_set_id > h264::MAX_PPS_ID + || pps.seq_parameter_set_id < 0 + || pps.seq_parameter_set_id > h264::MAX_SPS_ID) + { + return false; + } + + _tables.pps[pps.pic_parameter_set_id] = pps; + _tables.ppsValid[pps.pic_parameter_set_id] = true; + + return true; + } + + return false; + } + + bool h264SlicePoc(H264Tables& _tables, const uint8_t* _rbsp, uint32_t _size, int32_t& _poc) + { + h264::Bitstream bs; + bs.init(_rbsp, _size); + + h264::NALHeader nal = {}; + + if (!h264::read_nal_header(&nal, &bs) ) + { + return false; + } + + h264::Bitstream peek = bs; + peek.ue(); + peek.ue(); + const uint32_t ppsId = peek.ue(); + + if (ppsId > uint32_t(h264::MAX_PPS_ID) + || !_tables.ppsValid[ppsId]) + { + return false; + } + + const uint32_t spsId = uint32_t(_tables.pps[ppsId].seq_parameter_set_id); + + if (spsId > uint32_t(h264::MAX_SPS_ID) + || !_tables.spsValid[spsId]) + { + return false; + } + + h264::SliceHeader sh = {}; + h264::read_slice_header(&sh, &nal, _tables.pps.data(), _tables.sps.data(), &bs); + + const bool isIdr = h264::NAL_UNIT_TYPE_CODED_SLICE_IDR == nal.type; + _poc = computePoc(_tables.prevPocMsb, _tables.prevPocLsb, _tables.sps[spsId], sh, nal, isIdr); + + return true; + } + + struct HevcSps + { + bool valid; + bool separateColourPlane; + uint8_t chromaFormatIdc; + uint8_t bitDepth; + uint8_t profile; + uint8_t level; + uint16_t width; + uint16_t height; + uint32_t log2MaxPocLsb; + int64_t frameUs; + }; + + struct HevcPps + { + bool valid; + bool dependentSliceSegments; + bool outputFlagPresent; + uint32_t spsId; + uint32_t numExtraSliceHeaderBits; + }; + + struct HevcTables + { + HevcTables() + : prevPocMsb(0) + , prevPocLsb(0) + , firstPicture(true) + { + bx::memSet(sps, 0, sizeof(sps) ); + bx::memSet(pps, 0, sizeof(pps) ); + } + + HevcSps sps[hevc::kMaxSpsCount]; + HevcPps pps[hevc::kMaxPpsCount]; + int32_t prevPocMsb; + int32_t prevPocLsb; + bool firstPicture; + }; + + void skipHevcProfileTierLevel(h264::Bitstream& _bs, uint32_t _maxSubLayersMinus1, uint8_t* _profile, uint8_t* _level) + { + _bs.u(2); + _bs.u(1); + const uint32_t profileIdc = _bs.u(5); + _bs.u(32); + _bs.u(32); + _bs.u(16); + const uint32_t levelIdc = _bs.u(8); + + if (NULL != _profile) + { + *_profile = uint8_t(profileIdc); + *_level = uint8_t(levelIdc); + } + + bool subLayerProfilePresent[hevc::kMaxSubLayers] = {}; + bool subLayerLevelPresent[hevc::kMaxSubLayers] = {}; + + for (uint32_t ii = 0; ii < _maxSubLayersMinus1 && ii < hevc::kMaxSubLayers; ++ii) + { + subLayerProfilePresent[ii] = 0 != _bs.u1(); + subLayerLevelPresent[ii] = 0 != _bs.u1(); + } + + if (0 < _maxSubLayersMinus1) + { + for (uint32_t ii = _maxSubLayersMinus1; ii < 8; ++ii) + { + _bs.u(2); + } + } + + for (uint32_t ii = 0; ii < _maxSubLayersMinus1 && ii < hevc::kMaxSubLayers; ++ii) + { + if (subLayerProfilePresent[ii]) + { + _bs.u(32); + _bs.u(32); + _bs.u(24); + } + + if (subLayerLevelPresent[ii]) + { + _bs.u(8); + } + } + } + + void skipHevcScalingListData(h264::Bitstream& _bs) + { + for (uint32_t sizeId = 0; sizeId < 4; ++sizeId) + { + for (uint32_t matrixId = 0; matrixId < 6; matrixId += 3 == sizeId ? 3 : 1) + { + if (0 == _bs.u1() ) + { + _bs.ue(); + } + else + { + const uint32_t coefNum = bx::min(64, 1u << (4 + (sizeId << 1) ) ); + + if (1 < sizeId) + { + _bs.se(); + } + + for (uint32_t ii = 0; ii < coefNum; ++ii) + { + _bs.se(); + } + } + } + } + } + + void skipHevcStRefPicSet(h264::Bitstream& _bs, uint32_t _idx, uint32_t* _numDeltaPocs) + { + if (0 != _idx + && 0 != _bs.u1() ) + { + _bs.u1(); + _bs.ue(); + + const uint32_t refIdx = _idx - 1; + uint32_t count = 0; + + for (uint32_t jj = 0; jj <= _numDeltaPocs[refIdx]; ++jj) + { + const bool used = 0 != _bs.u1(); + const bool useDelta = used || 0 != _bs.u1(); + count += useDelta ? 1 : 0; + } + + _numDeltaPocs[_idx] = count; + return; + } + + const uint32_t numNegative = _bs.ue(); + const uint32_t numPositive = _bs.ue(); + + for (uint32_t ii = 0; ii < numNegative && ii < 64; ++ii) + { + _bs.ue(); + _bs.u1(); + } + + for (uint32_t ii = 0; ii < numPositive && ii < 64; ++ii) + { + _bs.ue(); + _bs.u1(); + } + + _numDeltaPocs[_idx] = bx::min(numNegative + numPositive, 64); + } + + bool parseHevcSps(const uint8_t* _rbsp, uint32_t _size, HevcSps& _out, uint32_t& _spsId) + { + h264::Bitstream bs; + bs.init(_rbsp + hevc::kNalHeaderSize, _size - hevc::kNalHeaderSize); + + _out = {}; + + bs.u(4); + const uint32_t maxSubLayersMinus1 = bs.u(3); + bs.u1(); + skipHevcProfileTierLevel(bs, maxSubLayersMinus1, &_out.profile, &_out.level); + + _spsId = bs.ue(); + + const uint32_t chromaFormatIdc = bs.ue(); + _out.chromaFormatIdc = uint8_t(chromaFormatIdc); + _out.separateColourPlane = 3 == chromaFormatIdc && 0 != bs.u1(); + + const uint32_t width = bs.ue(); + const uint32_t height = bs.ue(); + + uint32_t cropX = 0; + uint32_t cropY = 0; + + if (0 != bs.u1() ) + { + const uint32_t subWidthC = (1 == chromaFormatIdc || 2 == chromaFormatIdc) ? 2 : 1; + const uint32_t subHeightC = 1 == chromaFormatIdc ? 2 : 1; + const uint32_t left = bs.ue(); + const uint32_t right = bs.ue(); + const uint32_t top = bs.ue(); + const uint32_t bottom = bs.ue(); + cropX = (left + right) * subWidthC; + cropY = (top + bottom) * subHeightC; + } + + _out.bitDepth = uint8_t(bs.ue() + 8); + bs.ue(); + _out.log2MaxPocLsb = bs.ue() + 4; + + const bool subLayerOrderingInfoPresent = 0 != bs.u1(); + + for (uint32_t ii = subLayerOrderingInfoPresent ? 0 : maxSubLayersMinus1; ii <= maxSubLayersMinus1; ++ii) + { + bs.ue(); + bs.ue(); + bs.ue(); + } + + bs.ue(); + bs.ue(); + bs.ue(); + bs.ue(); + bs.ue(); + bs.ue(); + + if (0 != bs.u1() + && 0 != bs.u1() ) + { + skipHevcScalingListData(bs); + } + + bs.u1(); + bs.u1(); + + if (0 != bs.u1() ) + { + bs.u(4); + bs.u(4); + bs.ue(); + bs.ue(); + bs.u1(); + } + + const uint32_t numStRefPicSets = bs.ue(); + + if (hevc::kMaxStRefPicSets < numStRefPicSets) + { + return false; + } + + uint32_t numDeltaPocs[hevc::kMaxStRefPicSets] = {}; + + for (uint32_t ii = 0; ii < numStRefPicSets; ++ii) + { + skipHevcStRefPicSet(bs, ii, numDeltaPocs); + } + + if (0 != bs.u1() ) + { + const uint32_t numLongTerm = bs.ue(); + + for (uint32_t ii = 0; ii < numLongTerm && ii < 32; ++ii) + { + bs.u(int(_out.log2MaxPocLsb) ); + bs.u1(); + } + } + + bs.u1(); + bs.u1(); + + if (0 != bs.u1() ) + { + if (0 != bs.u1() ) + { + if (255 == bs.u(8) ) + { + bs.u(16); + bs.u(16); + } + } + + if (0 != bs.u1() ) + { + bs.u1(); + } + + if (0 != bs.u1() ) + { + bs.u(3); + bs.u1(); + + if (0 != bs.u1() ) + { + bs.u(8); + bs.u(8); + bs.u(8); + } + } + + if (0 != bs.u1() ) + { + bs.ue(); + bs.ue(); + } + + bs.u1(); + bs.u1(); + bs.u1(); + + if (0 != bs.u1() ) + { + bs.ue(); + bs.ue(); + bs.ue(); + bs.ue(); + } + + if (0 != bs.u1() ) + { + const uint32_t numUnitsInTick = bs.u(32); + const uint32_t timeScale = bs.u(32); + + if (0 != numUnitsInTick + && 0 != timeScale) + { + _out.frameUs = int64_t(1000000ull * uint64_t(numUnitsInTick) / uint64_t(timeScale) ); + } + } + } + + if (bs.eof() + || 0 == width + || 0 == height + || cropX >= width + || cropY >= height) + { + return false; + } + + _out.width = uint16_t(width - cropX); + _out.height = uint16_t(height - cropY); + _out.valid = true; + + return true; + } + + bool parseHevcPps(const uint8_t* _rbsp, uint32_t _size, HevcPps& _out, uint32_t& _ppsId) + { + h264::Bitstream bs; + bs.init(_rbsp + hevc::kNalHeaderSize, _size - hevc::kNalHeaderSize); + + _out = {}; + + _ppsId = bs.ue(); + _out.spsId = bs.ue(); + _out.dependentSliceSegments = 0 != bs.u1(); + _out.outputFlagPresent = 0 != bs.u1(); + _out.numExtraSliceHeaderBits = bs.u(3); + _out.valid = !bs.eof(); + + return _out.valid; + } + + bool parseHevcParameterSet(HevcTables& _tables, const uint8_t* _rbsp, uint32_t _size) + { + if (_size <= hevc::kNalHeaderSize) + { + return false; + } + + const uint8_t type = hevc::getNalUnitType(_rbsp); + + if (hevc::SPS_NUT == type) + { + HevcSps sps; + uint32_t id; + + if (parseHevcSps(_rbsp, _size, sps, id) + && id < hevc::kMaxSpsCount) + { + _tables.sps[id] = sps; + return true; + } + } + else if (hevc::PPS_NUT == type) + { + HevcPps pps; + uint32_t id; + + if (parseHevcPps(_rbsp, _size, pps, id) + && id < hevc::kMaxPpsCount) + { + _tables.pps[id] = pps; + return true; + } + } + + return false; + } + + bool hevcSlicePoc(HevcTables& _tables, const uint8_t* _rbsp, uint32_t _size, int32_t& _poc) + { + const uint8_t type = hevc::getNalUnitType(_rbsp); + + h264::Bitstream bs; + bs.init(_rbsp + hevc::kNalHeaderSize, _size - hevc::kNalHeaderSize); + + bs.u1(); + + if (hevc::isIrap(type) ) + { + bs.u1(); + } + + const uint32_t ppsId = bs.ue(); + + if (hevc::kMaxPpsCount <= ppsId + || !_tables.pps[ppsId].valid) + { + return false; + } + + const HevcPps& pps = _tables.pps[ppsId]; + + if (hevc::kMaxSpsCount <= pps.spsId + || !_tables.sps[pps.spsId].valid) + { + return false; + } + + const HevcSps& sps = _tables.sps[pps.spsId]; + + for (uint32_t ii = 0; ii < pps.numExtraSliceHeaderBits; ++ii) + { + bs.u1(); + } + + bs.ue(); + + if (pps.outputFlagPresent) + { + bs.u1(); + } + + if (sps.separateColourPlane) + { + bs.u(2); + } + + const int32_t maxPocLsb = 1 << sps.log2MaxPocLsb; + const int32_t pocLsb = hevc::isIdr(type) ? 0 : int32_t(bs.u(int(sps.log2MaxPocLsb) ) ); + + int32_t pocMsb; + + if (hevc::isIdrOrBla(type) + || (hevc::CRA_NUT == type && _tables.firstPicture) ) + { + pocMsb = 0; + } + else if (pocLsb < _tables.prevPocLsb + && _tables.prevPocLsb - pocLsb >= maxPocLsb / 2) + { + pocMsb = _tables.prevPocMsb + maxPocLsb; + } + else if (pocLsb > _tables.prevPocLsb + && pocLsb - _tables.prevPocLsb > maxPocLsb / 2) + { + pocMsb = _tables.prevPocMsb - maxPocLsb; + } + else + { + pocMsb = _tables.prevPocMsb; + } + + _tables.prevPocMsb = pocMsb; + _tables.prevPocLsb = pocLsb; + _tables.firstPicture = false; + + _poc = pocMsb + pocLsb; + + return true; + } + + struct Av1SequenceHeader + { + uint32_t maxWidth; + uint32_t maxHeight; + int64_t frameUs; + uint8_t profile; + uint8_t levelIdx; + uint8_t highBitdepth; + uint8_t twelveBit; + uint8_t monochrome; + uint8_t subsamplingX; + uint8_t subsamplingY; + uint8_t reducedStillPictureHeader; + }; + + uint32_t uvlc(h264::Bitstream& _bs) + { + uint32_t leadingZeros = 0; + + while (leadingZeros < av1::kMaxLeadingZeros + && 0 == _bs.u1() ) + { + ++leadingZeros; + } + + if (av1::kMaxLeadingZeros <= leadingZeros) + { + return UINT32_MAX; + } + + return _bs.u(int(leadingZeros) ) + (1u << leadingZeros) - 1; + } + + bool parseAv1SequenceHeader(const uint8_t* _data, uint32_t _size, Av1SequenceHeader& _out) + { + h264::Bitstream br; + br.init(_data, _size); + + _out = {}; + _out.profile = uint8_t(br.u(3) ); + br.u1(); + _out.reducedStillPictureHeader = uint8_t(br.u1() ); + + bool decoderModelInfoPresent = false; + uint32_t bufferDelayLength = 0; + + if (_out.reducedStillPictureHeader) + { + _out.levelIdx = uint8_t(br.u(5) ); + } + else + { + if (0 != br.u1() ) + { + const uint32_t numUnitsInDisplayTick = br.u(32); + const uint32_t timeScale = br.u(32); + uint32_t ticksPerPicture = 1; + + if (0 != br.u1() ) + { + ticksPerPicture = uvlc(br) + 1; + } + + if (0 != numUnitsInDisplayTick + && 0 != timeScale) + { + _out.frameUs = int64_t(1000000ull * uint64_t(numUnitsInDisplayTick) * ticksPerPicture / uint64_t(timeScale) ); + } + + decoderModelInfoPresent = 0 != br.u1(); + + if (decoderModelInfoPresent) + { + bufferDelayLength = br.u(5) + 1; + br.u(32); + br.u(5); + br.u(5); + } + } + + const bool initialDisplayDelayPresent = 0 != br.u1(); + const uint32_t numOperatingPoints = br.u(5) + 1; + + for (uint32_t ii = 0; ii < numOperatingPoints; ++ii) + { + br.u(12); + const uint32_t levelIdx = br.u(5); + + if (7 < levelIdx) + { + br.u1(); + } + + if (0 == ii) + { + _out.levelIdx = uint8_t(levelIdx); + } + + if (decoderModelInfoPresent + && 0 != br.u1() ) + { + br.u(int(bufferDelayLength) ); + br.u(int(bufferDelayLength) ); + br.u1(); + } + + if (initialDisplayDelayPresent + && 0 != br.u1() ) + { + br.u(4); + } + } + } + + const uint32_t widthBits = br.u(4) + 1; + const uint32_t heightBits = br.u(4) + 1; + _out.maxWidth = br.u(int(widthBits) ) + 1; + _out.maxHeight = br.u(int(heightBits) ) + 1; + + if (!_out.reducedStillPictureHeader + && 0 != br.u1() ) + { + br.u(4); + br.u(3); + } + + br.u(3); + + if (!_out.reducedStillPictureHeader) + { + br.u(4); + const bool enableOrderHint = 0 != br.u1(); + + if (enableOrderHint) + { + br.u(2); + } + + const uint32_t forceScreenContentTools = 0 != br.u1() ? 2 : br.u1(); + + if (0 < forceScreenContentTools + && 0 == br.u1() ) + { + br.u1(); + } + + if (enableOrderHint) + { + br.u(3); + } + } + + br.u(3); + + _out.highBitdepth = uint8_t(br.u1() ); + _out.twelveBit = av1::PROFILE_PROFESSIONAL == _out.profile && _out.highBitdepth ? uint8_t(br.u1() ) : 0; + _out.monochrome = av1::PROFILE_HIGH == _out.profile ? 0 : uint8_t(br.u1() ); + + uint32_t colorPrimaries = 2; + uint32_t transfer = 2; + uint32_t matrix = 2; + + if (0 != br.u1() ) + { + colorPrimaries = br.u(8); + transfer = br.u(8); + matrix = br.u(8); + } + + if (_out.monochrome) + { + br.u1(); + _out.subsamplingX = 1; + _out.subsamplingY = 1; + } + else if (1 == colorPrimaries && 13 == transfer && 0 == matrix) + { + } + else + { + br.u1(); + + if (av1::PROFILE_MAIN == _out.profile) + { + _out.subsamplingX = 1; + _out.subsamplingY = 1; + } + else if (av1::PROFILE_PROFESSIONAL == _out.profile) + { + if (_out.twelveBit) + { + _out.subsamplingX = uint8_t(br.u1() ); + _out.subsamplingY = _out.subsamplingX ? uint8_t(br.u1() ) : 0; + } + else + { + _out.subsamplingX = 1; + } + } + } + + return !br.eof() + && 0 != _out.maxWidth + && 0 != _out.maxHeight + ; + } + + void noteAv1SequenceHeader(Impl& _video, const Av1SequenceHeader& _seq) + { + _video.profile = _seq.profile; + _video.level = _seq.levelIdx; + _video.bitDepth = _seq.twelveBit ? 12 : _seq.highBitdepth ? 10 : 8; + _video.chroma = _seq.subsamplingX && _seq.subsamplingY ? 0 + : _seq.subsamplingX ? 2 + : 4 + ; + _video.width = uint16_t(_seq.maxWidth); + _video.height = uint16_t(_seq.maxHeight); + } + + struct Obu + { + uint8_t type; + uint32_t headerSize; + uint32_t payloadSize; + bool hasSize; + }; + + bool readObuHeader(const uint8_t* _data, uint32_t _size, Obu& _out) + { + if (0 == _size) + { + return false; + } + + const uint8_t header = _data[0]; + const bool hasExtension = 0 != (header & av1::kObuExtensionFlag); + + _out.type = (header >> av1::kObuTypeShift) & av1::kObuTypeMask; + _out.hasSize = 0 != (header & av1::kObuHasSizeField); + _out.headerSize = 1 + (hasExtension ? 1 : 0); + + if (0 != (header & 0x80) + || _size < _out.headerSize) + { + return false; + } + + if (!_out.hasSize) + { + _out.payloadSize = _size - _out.headerSize; + return true; + } + + uint32_t size = 0; + + for (uint32_t shift = 0; shift < 32; shift += 7) + { + if (_out.headerSize >= _size) + { + return false; + } + + const uint8_t byte = _data[_out.headerSize++]; + size |= uint32_t(byte & av1::kLeb128ValueMask) << shift; + + if (0 == (byte & av1::kLeb128ContinuationBit) ) + { + break; + } + } + + _out.payloadSize = bx::min(size, _size - _out.headerSize); + + return true; + } + + template + void visitObus(const uint8_t* _data, uint32_t _size, Fn _visit) + { + uint32_t pos = 0; + + while (pos < _size) + { + Obu obu; + + if (!readObuHeader(_data + pos, _size - pos, obu) ) + { + break; + } + + const uint32_t obuSize = obu.headerSize + obu.payloadSize; + _visit(obu.type, _data + pos, obuSize, _data + pos + obu.headerSize, obu.payloadSize); + pos += obuSize; + } + } + + bool av1FrameIsKey(const uint8_t* _payload, uint32_t _size, bool _reducedStillPictureHeader) + { + if (0 == _size) + { + return false; + } + + if (_reducedStillPictureHeader) + { + return true; + } + + h264::Bitstream br; + br.init(_payload, _size); + + if (0 != br.u1() ) + { + return false; + } + + const uint32_t frameType = br.u(2); + const bool showFrame = 0 != br.u1(); + + return av1::KEY_FRAME == frameType && showFrame; + } + + bool av1TemporalUnitIsKey(const uint8_t* _data, uint32_t _size, bool _reducedStillPictureHeader) + { + bool seen = false; + bool key = false; + + visitObus(_data, _size, [&](uint8_t _type, const uint8_t*, uint32_t, const uint8_t* _payload, uint32_t _payloadSize) + { + if (!seen + && (av1::OBU_FRAME_HEADER == _type || av1::OBU_FRAME == _type) ) + { + seen = true; + key = av1FrameIsKey(_payload, _payloadSize, _reducedStillPictureHeader); + } + }); + + return key; + } + + bool takeAv1SequenceHeader(Impl& _video, Av1SequenceHeader& _seq, const uint8_t* _data, uint32_t _size) + { + bool found = false; + + visitObus(_data, _size, [&](uint8_t _type, const uint8_t* _obu, uint32_t _obuSize, const uint8_t* _payload, uint32_t _payloadSize) + { + if (!found + && av1::OBU_SEQUENCE_HEADER == _type + && parseAv1SequenceHeader(_payload, _payloadSize, _seq) ) + { + found = true; + _video.parameterSets.clear(); + _video.parameterSets.insert(_video.parameterSets.end(), _obu, _obu + _obuSize); + } + }); + + return found; + } + + struct Probe + { + Impl::Container container; + bgfx::VideoCodec::Enum codec; + }; + + bool probeAnnexB(const uint8_t* _data, uint32_t _size, bgfx::VideoCodec::Enum& _codec) + { + uint32_t header = 0; + + if (4 <= _size && 0 == _data[0] && 0 == _data[1] && 0 == _data[2] && 1 == _data[3]) + { + header = 4; + } + else if (3 <= _size && 0 == _data[0] && 0 == _data[1] && 1 == _data[2]) + { + header = 3; + } + else + { + return false; + } + + if (_size < header + 2 + || 0 != (_data[header] & 0x80) ) + { + return false; + } + + const uint8_t hevcType = hevc::getNalUnitType(_data + header); + const uint32_t hevcLayerId = (uint32_t(_data[header] & 1) << 5) | (_data[header + 1] >> 3); + const uint32_t hevcTidPlus1 = _data[header + 1] & 7; + + const bool hevcHead = false + || hevc::isParameterSet(hevcType) + || hevc::AUD_NUT == hevcType + || hevc::PREFIX_SEI_NUT == hevcType + || hevc::isIrap(hevcType) + ; + + if (hevcHead + && 0 == hevcLayerId + && 0 != hevcTidPlus1) + { + _codec = bgfx::VideoCodec::H265; + return true; + } + + const uint8_t h264Type = _data[header] & kH264NalUnitTypeMask; + + const bool h264Head = false + || h264::NAL_UNIT_TYPE_SPS == h264Type + || h264::NAL_UNIT_TYPE_PPS == h264Type + || h264::NAL_UNIT_TYPE_AUD == h264Type + || h264::NAL_UNIT_TYPE_SEI == h264Type + || h264::NAL_UNIT_TYPE_CODED_SLICE_IDR == h264Type + ; + + if (h264Head) + { + _codec = bgfx::VideoCodec::H264; + return true; + } + + return false; + } + + bool probeObu(const uint8_t* _data, uint32_t _size) + { + Obu obu; + + if (!readObuHeader(_data, _size, obu) + || !obu.hasSize) + { + return false; + } + + if (av1::OBU_SEQUENCE_HEADER == obu.type) + { + return true; + } + + if (av1::OBU_TEMPORAL_DELIMITER != obu.type + || 0 != obu.payloadSize) + { + return false; + } + + const uint32_t next = obu.headerSize; + + return readObuHeader(_data + next, _size - next, obu) + && obu.hasSize + && av1::OBU_SEQUENCE_HEADER == obu.type + ; + } + + bool probeBytes(const uint8_t* _data, uint32_t _size, Probe& _out) + { + if (12 <= _size) + { + const uint8_t* box = _data + 4; + + if (isFourCC(box, "ftyp") + || isFourCC(box, "moov") + || isFourCC(box, "mdat") + || isFourCC(box, "free") + || isFourCC(box, "skip") + || isFourCC(box, "wide") ) + { + _out = { Impl::Mp4, bgfx::VideoCodec::Count }; + return true; + } + + if (isFourCC(_data, "DKIF") + && isFourCC(_data + 8, "AV01") ) + { + _out = { Impl::Ivf, bgfx::VideoCodec::AV1 }; + return true; + } + } + + bgfx::VideoCodec::Enum codec; + + if (probeAnnexB(_data, _size, codec) ) + { + _out = { Impl::AnnexB, codec }; + return true; + } + + if (probeObu(_data, _size) ) + { + _out = { Impl::Obu, bgfx::VideoCodec::AV1 }; + return true; + } + + return false; + } + + bool probeFile(const bx::FilePath& _filePath, Probe& _out) + { + bx::FileReader reader; + bx::Error err; + + if (!bx::open(&reader, _filePath, &err) ) + { + return false; + } + + uint8_t data[kProbeSize] = {}; + const int32_t size = bx::read(&reader, data, sizeof(data), &err); + bx::close(&reader); + + return 0 < size + && probeBytes(data, uint32_t(size), _out) + ; + } + + class FileScanner + { + public: + FileScanner(bx::FileReaderI* _reader, uint64_t _size) + : m_reader(_reader) + , m_size(_size) + , m_bufferPos(0) + , m_bufferLen(0) + { + m_buffer.resize(kScanBufferSize); + } + + const uint8_t* window(uint64_t _pos, uint32_t _minLen, uint32_t& _len) + { + if (_pos >= m_size) + { + _len = 0; + return NULL; + } + + const uint64_t want = bx::min(_minLen, m_size - _pos); + + if (_pos < m_bufferPos + || _pos + want > m_bufferPos + m_bufferLen) + { + refill(_pos); + } + + _len = uint32_t(bx::min(m_bufferPos + m_bufferLen - _pos, UINT32_MAX) ); + + return &m_buffer[size_t(_pos - m_bufferPos)]; + } + + uint64_t size() const + { + return m_size; + } + + private: + void refill(uint64_t _pos) + { + bx::Error err; + bx::seek(m_reader, int64_t(_pos), bx::Whence::Begin); + const int32_t len = bx::read(m_reader, m_buffer.data(), int32_t(m_buffer.size() ), &err); + + m_bufferPos = _pos; + m_bufferLen = 0 < len ? uint32_t(len) : 0; + } + + bx::FileReaderI* m_reader; + uint64_t m_size; + uint64_t m_bufferPos; + uint32_t m_bufferLen; + stl::vector m_buffer; + }; + + uint64_t findStartCode(FileScanner& _scanner, uint64_t _pos) + { + const uint64_t size = _scanner.size(); + + while (_pos + 3 <= size) + { + uint32_t len; + const uint8_t* data = _scanner.window(_pos, 3, len); + + if (NULL == data + || len < 3) + { + break; + } + + uint32_t ii = 0; + + for (; ii + 3 <= len; ++ii) + { + if (0 == data[ii] + && 0 == data[ii+1] + && 1 == data[ii+2]) + { + const uint64_t sc = _pos + ii; + + if (0 < ii) + { + return 0 == data[ii-1] ? sc - 1 : sc; + } + + if (0 < sc) + { + uint32_t prevLen; + const uint8_t* prev = _scanner.window(sc - 1, 1, prevLen); + return NULL != prev && 0 == prev[0] ? sc - 1 : sc; + } + + return sc; + } + } + + _pos += ii; + } + + return size; + } + + struct GopPicture + { + int32_t poc; + uint32_t frame; + }; + + int32_t compareGopPicture(const void* _lhs, const void* _rhs) + { + const GopPicture& lhs = *(const GopPicture*)_lhs; + const GopPicture& rhs = *(const GopPicture*)_rhs; + + if (lhs.poc != rhs.poc) + { + return lhs.poc < rhs.poc ? -1 : 1; + } + + return lhs.frame < rhs.frame ? -1 : lhs.frame > rhs.frame ? 1 : 0; + } + + void finalizeGop(Impl& _video, stl::vector& _gop, uint64_t& _displayBase, int64_t _frameUs) + { + if (_gop.empty() ) + { + return; + } + + bx::quickSort(_gop.data(), uint32_t(_gop.size() ), sizeof(GopPicture), compareGopPicture); + + for (uint32_t ii = 0; ii < uint32_t(_gop.size() ); ++ii) + { + _video.frames[_gop[ii].frame].ptsUs = (_displayBase + ii) * uint64_t(_frameUs); + } + + _displayBase += _gop.size(); + _gop.clear(); + } + + void closeFrame(Impl& _video, uint64_t _end) + { + if (!_video.frames.empty() ) + { + Impl::Frame& last = _video.frames.back(); + last.size = uint32_t(_end - last.offset); + _video.totalBytes += last.size; + } + } + + bool indexAnnexB(Impl& _video) + { + const bool isHevc = bgfx::VideoCodec::H265 == _video.codec; + + FileScanner scanner(&_video.reader, _video.fileSize); + + H264Tables h264Tables; + HevcTables hevcTables; + + int64_t frameUs = 0; + uint64_t displayBase = 0; + uint64_t unitBegin = UINT64_MAX; + uint64_t pos = findStartCode(scanner, 0); + + stl::vector gop; + stl::vector rbsp; + rbsp.resize(kNalPeekSize); + + while (pos < scanner.size() ) + { + uint32_t len; + const uint8_t* sc = scanner.window(pos, 4, len); + + if (NULL == sc + || len < 3) + { + break; + } + + const uint32_t scLen = 1 == sc[2] ? 3 : 4; + const uint64_t header = pos + scLen; + const uint64_t next = findStartCode(scanner, header); + + if (header >= next) + { + pos = next; + continue; + } + + const uint32_t peek = uint32_t(bx::min(next - header, kNalPeekSize) ); + const uint8_t* nal = scanner.window(header, peek, len); + const uint32_t nalLen = bx::min(peek, len); + const uint32_t rbspSize = stripEmulationPrevention(rbsp.data(), nal, nalLen); + const uint32_t minSize = isHevc ? hevc::kNalHeaderSize + 1 : 2; + + if (rbspSize < minSize) + { + pos = next; + continue; + } + + bool parameterSet; + bool slice; + bool firstSlice; + bool keyframe; + + if (isHevc) + { + const uint8_t type = hevc::getNalUnitType(rbsp.data() ); + parameterSet = hevc::isParameterSet(type); + slice = hevc::isVcl(type); + firstSlice = 0 != (rbsp[hevc::kNalHeaderSize] & 0x80); + keyframe = hevc::isIrap(type); + } + else + { + const uint8_t type = rbsp[0] & kH264NalUnitTypeMask; + parameterSet = h264::NAL_UNIT_TYPE_SPS == type || h264::NAL_UNIT_TYPE_PPS == type; + slice = h264::NAL_UNIT_TYPE_CODED_SLICE_NON_IDR == type || h264::NAL_UNIT_TYPE_CODED_SLICE_IDR == type; + firstSlice = 0 != (rbsp[1] & 0x80); + keyframe = h264::NAL_UNIT_TYPE_CODED_SLICE_IDR == type; + } + + if (parameterSet) + { + if (isHevc) + { + parseHevcParameterSet(hevcTables, rbsp.data(), rbspSize); + } + else + { + parseH264ParameterSet(h264Tables, _video, frameUs, rbsp.data(), rbspSize); + } + + if (_video.frames.empty() + && UINT64_MAX == unitBegin) + { + appendNalAsAnnexB(_video.parameterSets, nal, nalLen); + } + else + { + unitBegin = bx::min(unitBegin, pos); + } + } + else if (slice && firstSlice) + { + const uint64_t start = UINT64_MAX != unitBegin ? unitBegin : pos; + closeFrame(_video, start); + + int32_t poc = 0; + + const bool haveOrder = isHevc + ? hevcSlicePoc(hevcTables, rbsp.data(), rbspSize, poc) + : h264SlicePoc(h264Tables, rbsp.data(), rbspSize, poc) + ; + + const bool gopStart = isHevc + ? hevc::isIdrOrBla(hevc::getNalUnitType(rbsp.data() ) ) + : keyframe + ; + + if (gopStart) + { + finalizeGop(_video, gop, displayBase, 0 != frameUs ? frameUs : kDefaultFrameUs); + } + + const uint32_t frameIdx = uint32_t(_video.frames.size() ); + + _video.frames.push_back( + { + .ptsUs = 0, + .offset = start, + .size = 0, + .isKeyframe = keyframe, + }); + + gop.push_back({ haveOrder ? poc : int32_t(frameIdx), frameIdx }); + unitBegin = UINT64_MAX; + } + else if (!slice + && !_video.frames.empty() ) + { + unitBegin = bx::min(unitBegin, pos); + } + + pos = next; + } + + closeFrame(_video, scanner.size() ); + + if (0 == frameUs + && isHevc) + { + for (uint32_t ii = 0; ii < hevc::kMaxSpsCount; ++ii) + { + if (hevcTables.sps[ii].valid) + { + const HevcSps& sps = hevcTables.sps[ii]; + _video.profile = sps.profile; + _video.level = sps.level; + _video.chroma = chromaFromFormatIdc(sps.chromaFormatIdc); + _video.bitDepth = sps.bitDepth; + _video.width = sps.width; + _video.height = sps.height; + frameUs = sps.frameUs; + break; + } + } + } + + finalizeGop(_video, gop, displayBase, 0 != frameUs ? frameUs : kDefaultFrameUs); + + return !_video.frames.empty() + && !_video.parameterSets.empty() + && 0 != _video.width + && 0 != _video.height + ; + } + + bool indexObu(Impl& _video) + { + FileScanner scanner(&_video.reader, _video.fileSize); + + Av1SequenceHeader seq = {}; + bool haveSeq = false; + bool keySeen = false; + uint64_t pos = 0; + + while (pos < scanner.size() ) + { + uint32_t len; + const uint8_t* data = scanner.window(pos, av1::kObuHeaderMaxSize, len); + Obu obu; + + if (NULL == data + || !readObuHeader(data, len, obu) ) + { + break; + } + + const uint64_t payload = pos + obu.headerSize; + const uint64_t next = obu.hasSize + ? bx::min(payload + obu.payloadSize, scanner.size() ) + : scanner.size() + ; + + if (av1::OBU_TEMPORAL_DELIMITER == obu.type + || _video.frames.empty() ) + { + closeFrame(_video, pos); + + _video.frames.push_back( + { + .ptsUs = 0, + .offset = pos, + .size = 0, + .isKeyframe = false, + }); + + keySeen = false; + } + + if (av1::OBU_SEQUENCE_HEADER == obu.type + && !haveSeq) + { + const uint32_t obuSize = uint32_t(next - pos); + const uint8_t* whole = scanner.window(pos, obuSize, len); + + if (NULL != whole + && obuSize <= len + && parseAv1SequenceHeader(whole + obu.headerSize, obu.payloadSize, seq) ) + { + haveSeq = true; + _video.parameterSets.insert(_video.parameterSets.end(), whole, whole + obuSize); + } + } + else if (!keySeen + && (av1::OBU_FRAME_HEADER == obu.type || av1::OBU_FRAME == obu.type) ) + { + const uint8_t* head = scanner.window(payload, 1, len); + keySeen = true; + _video.frames.back().isKeyframe = NULL != head && av1FrameIsKey(head, 1, 0 != seq.reducedStillPictureHeader); + } + + pos = next; + } + + closeFrame(_video, scanner.size() ); + + if (!haveSeq) + { + return false; + } + + noteAv1SequenceHeader(_video, seq); + + const int64_t frameUs = 0 != seq.frameUs ? seq.frameUs : kDefaultFrameUs; + + for (uint32_t ii = 0; ii < uint32_t(_video.frames.size() ); ++ii) + { + _video.frames[ii].ptsUs = uint64_t(ii) * uint64_t(frameUs); + } + + return !_video.frames.empty(); + } + + bool indexIvf(Impl& _video) + { + FileScanner scanner(&_video.reader, _video.fileSize); + + uint32_t len; + const uint8_t* header = scanner.window(0, ivf::kHeaderSize, len); + + if (NULL == header + || len < ivf::kHeaderSize) + { + return false; + } + + const uint32_t headerSize = readLe(header + 6); + const uint32_t timebaseDen = readLe(header + 16); + const uint32_t timebaseNum = readLe(header + 20); + + Av1SequenceHeader seq = {}; + bool haveSeq = false; + + uint64_t pos = bx::max(headerSize, ivf::kHeaderSize); + + while (pos + ivf::kFrameHeaderSize <= scanner.size() ) + { + const uint8_t* frameHeader = scanner.window(pos, ivf::kFrameHeaderSize, len); + + if (NULL == frameHeader + || len < ivf::kFrameHeaderSize) + { + break; + } + + const uint32_t size = readLe(frameHeader); + const uint64_t pts = readLe(frameHeader + 4); + const uint64_t offset = pos + ivf::kFrameHeaderSize; + + if (0 == size + || offset + size > scanner.size() + || kScanBufferSize < size) + { + break; + } + + const uint8_t* data = scanner.window(offset, size, len); + + if (NULL == data + || len < size) + { + break; + } + + if (!haveSeq) + { + haveSeq = takeAv1SequenceHeader(_video, seq, data, size); + } + + const uint64_t ptsUs = 0 != timebaseDen + ? pts * uint64_t(timebaseNum) * 1000000ull / uint64_t(timebaseDen) + : uint64_t(_video.frames.size() ) * uint64_t(kDefaultFrameUs) + ; + + _video.frames.push_back( + { + .ptsUs = ptsUs, + .offset = offset, + .size = size, + .isKeyframe = av1TemporalUnitIsKey(data, size, 0 != seq.reducedStillPictureHeader), + }); + + _video.totalBytes += size; + pos = offset + size; + } + + if (!haveSeq) + { + return false; + } + + noteAv1SequenceHeader(_video, seq); + + return !_video.frames.empty(); + } + + bool parseAvcC(Impl& _video, const uint8_t* _data, uint32_t _size, uint32_t& _lengthSize) + { if (NULL == _data || _size < kBoxHeaderSize + 7) { @@ -159,19 +2055,20 @@ namespace uint32_t pos = kBoxHeaderSize; ++pos; - const uint8_t profileIdc = _data[pos++]; + const uint8_t profileIdc = _data[pos++]; ++pos; - const uint8_t levelIdc = _data[pos++]; - _lengthSize = uint32_t(_data[pos++] & 0x03) + 1; - const uint32_t numSps = uint32_t(_data[pos++] & 0x1f); + const uint8_t levelIdc = _data[pos++]; + _lengthSize = uint32_t(_data[pos++] & 0x03) + 1; + const uint32_t numSps = uint32_t(_data[pos++] & 0x1f); _video.profile = profileIdc; _video.level = levelIdc; _video.chroma = 0; _video.bitDepth = 8; - bool spsParsed = false; + H264Tables tables; + int64_t frameUs = 0; stl::vector rbsp; for (uint32_t ii = 0; ii < numSps; ++ii) @@ -181,7 +2078,7 @@ namespace return false; } - const uint32_t nalLen = (uint32_t(_data[pos]) << 8) | uint32_t(_data[pos + 1]); + const uint32_t nalLen = readBe(_data + pos); pos += 2; if (pos + nalLen > _size) @@ -190,33 +2087,8 @@ namespace } appendNalAsAnnexB(_video.parameterSets, _data + pos, nalLen); - - if (!spsParsed - && nalLen >= 1) - { - stripEmulationPrevention(rbsp, _data + pos, nalLen); - - h264::Bitstream bs; - bs.init(rbsp.data(), rbsp.size() ); - - h264::NALHeader nal; - - if (h264::read_nal_header(&nal, &bs) - && h264::NAL_UNIT_TYPE_SPS == nal.type) - { - h264::SPS sps = {}; - h264::read_sps(&sps, &bs); - - _video.chroma = - 3 == sps.chroma_format_idc ? 4 - : 2 == sps.chroma_format_idc ? 2 - : 0 - ; - - _video.bitDepth = uint8_t(sps.bit_depth_luma_minus8 + 8); - spsParsed = true; - } - } + stripEmulationPrevention(rbsp, _data + pos, nalLen); + parseH264ParameterSet(tables, _video, frameUs, rbsp.data(), uint32_t(rbsp.size() ) ); pos += nalLen; } @@ -235,7 +2107,7 @@ namespace return false; } - const uint32_t nalLen = (uint32_t(_data[pos]) << 8) | uint32_t(_data[pos + 1]); + const uint32_t nalLen = readBe(_data + pos); pos += 2; if (pos + nalLen > _size) @@ -250,10 +2122,8 @@ namespace return true; } - bool parseHvcC(VideoPlayer::Impl& _video, const uint8_t* _data, uint32_t _size, uint32_t& _lengthSize) + bool parseHvcC(Impl& _video, const uint8_t* _data, uint32_t _size, uint32_t& _lengthSize) { - const uint32_t kBoxHeaderSize = 8; - if (NULL == _data || _size < kBoxHeaderSize + 23) { @@ -263,8 +2133,7 @@ namespace uint32_t pos = kBoxHeaderSize; ++pos; - const uint8_t profileByte = _data[pos++]; - const uint8_t profileIdc = profileByte & 0x1f; + const uint8_t profileIdc = _data[pos++] & 0x1f; pos += 10; const uint8_t levelIdc = _data[pos++]; @@ -274,8 +2143,7 @@ namespace const uint8_t bitDepthLuma = uint8_t( (_data[pos++] & 0x07) + 8); pos += 3; - const uint8_t lengthSizeByte = _data[pos++]; - _lengthSize = uint32_t(lengthSizeByte & 0x03) + 1; + _lengthSize = uint32_t(_data[pos++] & 0x03) + 1; if (pos + 1 > _size) { @@ -284,13 +2152,9 @@ namespace const uint32_t numArrays = _data[pos++]; - _video.profile = profileIdc; - _video.level = levelIdc; - _video.chroma = - 3 == chromaFormat ? 4 - : 2 == chromaFormat ? 2 - : 0 - ; + _video.profile = profileIdc; + _video.level = levelIdc; + _video.chroma = chromaFromFormatIdc(chromaFormat); _video.bitDepth = bitDepthLuma; stl::vector vps; @@ -304,19 +2168,18 @@ namespace return false; } - const uint8_t arrayHdr = _data[pos++]; - const uint8_t nalUnitType = arrayHdr & 0x3f; - const uint32_t numNalus = (uint32_t(_data[pos]) << 8) | uint32_t(_data[pos + 1]); + const uint8_t nalUnitType = _data[pos++] & 0x3f; + const uint32_t numNalus = readBe(_data + pos); pos += 2; stl::vector* dst = NULL; switch (nalUnitType) { - case 32: dst = &vps; break; - case 33: dst = &sps; break; - case 34: dst = &pps; break; - default: break; + case hevc::VPS_NUT: dst = &vps; break; + case hevc::SPS_NUT: dst = &sps; break; + case hevc::PPS_NUT: dst = &pps; break; + default: break; } for (uint32_t nn = 0; nn < numNalus; ++nn) @@ -326,7 +2189,7 @@ namespace return false; } - const uint32_t nalLen = (uint32_t(_data[pos]) << 8) | uint32_t(_data[pos + 1]); + const uint32_t nalLen = readBe(_data + pos); pos += 2; if (pos + nalLen > _size) @@ -350,7 +2213,158 @@ namespace return true; } - bool loadVideoFile(VideoPlayer::Impl& _video, const char* _path) + bool parseAv1C(Impl& _video, const uint8_t* _data, uint32_t _size) + { + if (NULL == _data + || _size < kBoxHeaderSize + av1::kAv1CSize) + { + return false; + } + + const uint8_t* av1c = _data + kBoxHeaderSize; + + const uint8_t highBitdepth = (av1c[2] >> 6) & 1; + const uint8_t twelveBit = (av1c[2] >> 5) & 1; + const uint8_t subsamplingX = (av1c[2] >> 3) & 1; + const uint8_t subsamplingY = (av1c[2] >> 2) & 1; + + _video.profile = av1c[1] >> 5; + _video.level = av1c[1] & 0x1f; + _video.bitDepth = twelveBit ? 12 : highBitdepth ? 10 : 8; + _video.chroma = subsamplingX && subsamplingY ? 0 + : subsamplingX ? 2 + : 4 + ; + + Av1SequenceHeader seq; + takeAv1SequenceHeader(_video, seq, av1c + av1::kAv1CSize, _size - kBoxHeaderSize - av1::kAv1CSize); + + return true; + } + + lsmash_codec_specific_t* getUnstructuredCodecSpecific(lsmash_summary_t* _sum, uint32_t _idx, lsmash_codec_specific_t*& _cs) + { + _cs = lsmash_get_codec_specific_data(_sum, _idx + 1); + + if (NULL == _cs) + { + return NULL; + } + + return LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED == _cs->format + ? _cs + : lsmash_convert_codec_specific_format(_cs, LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED) + ; + } + + void parseMp4CodecSpecific(Impl& _video, lsmash_summary_t* _sum, uint32_t& _lengthSize) + { + for (uint32_t ii = 0; ; ++ii) + { + lsmash_codec_specific_t* cs; + lsmash_codec_specific_t* raw = getUnstructuredCodecSpecific(_sum, ii, cs); + + if (NULL == cs) + { + break; + } + + bool done = false; + + if (NULL != raw) + { + const uint8_t* data = raw->data.unstructured; + const uint32_t size = raw->size; + + if (bgfx::VideoCodec::H264 == _video.codec + && LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_H264 == cs->type) + { + done = parseAvcC(_video, data, size, _lengthSize); + } + else if (bgfx::VideoCodec::H265 == _video.codec + && LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_HEVC == cs->type) + { + done = parseHvcC(_video, data, size, _lengthSize); + } + else if (bgfx::VideoCodec::AV1 == _video.codec + && LSMASH_CODEC_SPECIFIC_DATA_TYPE_UNKNOWN == cs->type + && kBoxHeaderSize <= size + && isFourCC(data + 4, "av1C") ) + { + done = parseAv1C(_video, data, size); + } + } + + if (raw != cs + && NULL != raw) + { + lsmash_destroy_codec_specific_data(raw); + } + + if (done) + { + break; + } + } + } + + void takeInBandParameterSets(Impl& _video) + { + lsmash_sample_t* s = lsmash_get_sample_from_media_timeline(_video.root, _video.trackId, 1); + + if (NULL == s) + { + return; + } + + if (bgfx::VideoCodec::AV1 == _video.codec) + { + Av1SequenceHeader seq; + takeAv1SequenceHeader(_video, seq, s->data, s->length); + } + else + { + const bool isHevc = bgfx::VideoCodec::H265 == _video.codec; + H264Tables tables; + int64_t frameUs = 0; + stl::vector rbsp; + + visitLengthPrefixedNals(s->data, s->length, _video.lengthSize, [&](const uint8_t* _nal, uint32_t _nalLen) + { + if (0 == _nalLen) + { + return; + } + + bool keep; + + if (isHevc) + { + keep = hevc::kNalHeaderSize <= _nalLen && hevc::isParameterSet(hevc::getNalUnitType(_nal) ); + } + else + { + const uint8_t type = _nal[0] & kH264NalUnitTypeMask; + keep = h264::NAL_UNIT_TYPE_SPS == type || h264::NAL_UNIT_TYPE_PPS == type; + + if (keep) + { + stripEmulationPrevention(rbsp, _nal, _nalLen); + parseH264ParameterSet(tables, _video, frameUs, rbsp.data(), uint32_t(rbsp.size() ) ); + } + } + + if (keep) + { + appendNalAsAnnexB(_video.parameterSets, _nal, _nalLen); + } + }); + } + + lsmash_delete_sample(s); + } + + bool loadMp4(Impl& _video, const char* _path) { lsmash_root_t* root = lsmash_create_root(); @@ -367,21 +2381,15 @@ namespace return false; } + _video.root = root; + _video.fp = fp; + _video.fpOpened = true; + lsmash_file_t* fh = lsmash_set_file(root, &fp); - if (NULL == fh) + if (NULL == fh + || lsmash_read_file(fh, &fp) < 0) { - lsmash_close_file(&fp); - lsmash_destroy_root(root); - - return false; - } - - if (lsmash_read_file(fh, &fp) < 0) - { - lsmash_close_file(&fp); - lsmash_destroy_root(root); - return false; } @@ -390,18 +2398,17 @@ namespace if (lsmash_get_movie_parameters(root, &mp) < 0) { - lsmash_close_file(&fp); - lsmash_destroy_root(root); - return false; } uint32_t trackId = 0; lsmash_media_parameters_t md; + lsmash_initialize_media_parameters(&md); for (uint32_t ii = 0; ii < mp.number_of_tracks; ++ii) { const uint32_t tid = lsmash_get_track_ID(root, ii + 1); + if (0 == tid) { continue; @@ -423,8 +2430,6 @@ namespace if (0 == trackId) { - lsmash_close_file(&fp); - lsmash_destroy_root(root); return false; } @@ -432,28 +2437,28 @@ namespace if (NULL == sum) { - lsmash_close_file(&fp); - lsmash_destroy_root(root); - return false; } if (lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_AVC1_VIDEO) + || lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_AVC2_VIDEO) || lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_AVC3_VIDEO) - ) + || lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_AVC4_VIDEO) ) { _video.codec = bgfx::VideoCodec::H264; } - else if (lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_HVC1_VIDEO) ) + else if (lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_HVC1_VIDEO) + || lsmash_check_codec_type_identical(sum->sample_type, ISOM_CODEC_TYPE_HEV1_VIDEO) ) { _video.codec = bgfx::VideoCodec::H265; } + else if (LSMASH_4CC('a', 'v', '0', '1') == sum->sample_type.fourcc) + { + _video.codec = bgfx::VideoCodec::AV1; + } else { lsmash_cleanup_summary(sum); - lsmash_close_file(&fp); - lsmash_destroy_root(root); - return false; } @@ -461,99 +2466,28 @@ namespace _video.width = uint16_t(vsum->width); _video.height = uint16_t(vsum->height); - _video.parameterSets.clear(); uint32_t lengthSize = 4; + parseMp4CodecSpecific(_video, sum, lengthSize); + lsmash_cleanup_summary(sum); - if (_video.codec == bgfx::VideoCodec::H264) - { - for (uint32_t ii = 0; ; ++ii) - { - lsmash_codec_specific_t* cs = lsmash_get_codec_specific_data(sum, ii + 1); - - if (NULL == cs) - { - break; - } - - if (cs->type != LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_H264) - { - continue; - } - - lsmash_codec_specific_t* csRaw = (cs->format == LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED) - ? cs - : lsmash_convert_codec_specific_format(cs, LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED) - ; - - if (NULL == csRaw) - { - continue; - } - - parseAvcC(_video, csRaw->data.unstructured, csRaw->size, lengthSize); - - if (csRaw != cs) - { - lsmash_destroy_codec_specific_data(csRaw); - } - - break; - } - } - - if (_video.codec == bgfx::VideoCodec::H265) - { - for (uint32_t ii = 0; ; ++ii) - { - lsmash_codec_specific_t* cs = lsmash_get_codec_specific_data(sum, ii + 1); - - if (NULL == cs) - { - break; - } - - if (cs->type != LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_HEVC) - { - continue; - } - - lsmash_codec_specific_t* csRaw = (cs->format == LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED) - ? cs - : lsmash_convert_codec_specific_format(cs, LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED) - ; - - if (NULL == csRaw) - { - continue; - } - - parseHvcC(_video, csRaw->data.unstructured, csRaw->size, lengthSize); - - if (csRaw != cs) - { - lsmash_destroy_codec_specific_data(csRaw); - } - - break; - } - } - - _video.maxDpbSlots = 16; - _video.maxActiveReferences = 4; + _video.trackId = trackId; + _video.lengthSize = lengthSize; if (lsmash_construct_timeline(root, trackId) < 0) { - lsmash_cleanup_summary(sum); - lsmash_close_file(&fp); - lsmash_destroy_root(root); - return false; } + if (_video.parameterSets.empty() ) + { + takeInBandParameterSets(_video); + } + const uint32_t timescale = bx::max(md.timescale, 1); const uint32_t numSamples = lsmash_get_sample_count_in_media_timeline(root, trackId); _video.frames.reserve(numSamples); + for (uint32_t ii = 0; ii < numSamples; ++ii) { lsmash_sample_t s = {}; @@ -563,30 +2497,60 @@ namespace break; } - VideoPlayer::Impl::Frame frame = {}; - frame.ptsUs = s.cts * 1000000ull / timescale; - frame.isKeyframe = 0 != (s.prop.ra_flags & ISOM_SAMPLE_RANDOM_ACCESS_FLAG_SYNC); + _video.frames.push_back( + { + .ptsUs = s.cts * 1000000ull / timescale, + .offset = 0, + .size = s.length, + .isKeyframe = 0 != (s.prop.ra_flags & ISOM_SAMPLE_RANDOM_ACCESS_FLAG_SYNC), + }); - _video.frames.push_back(frame); + _video.totalBytes += s.length; } - lsmash_cleanup_summary(sum); + return !_video.frames.empty() + && !_video.parameterSets.empty() + ; + } - if (_video.frames.empty() ) + bool loadVideoFile(Impl& _video, const char* _path) + { + const bx::FilePath filePath(_path); + Probe probe; + + if (!probeFile(filePath, probe) ) { - lsmash_close_file(&fp); - lsmash_destroy_root(root); - return false; } - _video.root = root; - _video.fp = fp; - _video.fpOpened = true; - _video.trackId = trackId; - _video.lengthSize = lengthSize; + _video.container = probe.container; + _video.codec = probe.codec; - return true; + if (Impl::Mp4 == probe.container) + { + return loadMp4(_video, _path); + } + + bx::Error err; + + if (!bx::open(&_video.reader, filePath, &err) ) + { + return false; + } + + _video.readerOpened = true; + _video.fileSize = uint64_t(bx::seek(&_video.reader, 0, bx::Whence::End) ); + bx::seek(&_video.reader, 0, bx::Whence::Begin); + + switch (probe.container) + { + case Impl::AnnexB: return indexAnnexB(_video); + case Impl::Obu: return indexObu(_video); + case Impl::Ivf: return indexIvf(_video); + default: break; + } + + return false; } } // namespace @@ -617,11 +2581,10 @@ VideoPlayer::~VideoPlayer() close(); } -bool VideoPlayer::isVideoExt(const bx::StringView& _ext) +bool VideoPlayer::isVideoFile(const bx::FilePath& _filePath) { - return 0 == bx::strCmpI(_ext, "mp4") - || 0 == bx::strCmpI(_ext, "m4v") - ; + Probe probe; + return probeFile(_filePath, probe); } bool VideoPlayer::open(const char* _filePath) @@ -649,18 +2612,13 @@ bool VideoPlayer::open(const char* _filePath) m_height = m_impl->height; const bgfx::Caps* caps = bgfx::getCaps(); - const uint32_t codecCaps = caps->codecs[m_impl->codec]; - const bool codecAvailable = true - && 0 != (caps->supported & BGFX_CAPS_VIDEO_DECODE) - && 0 != (codecCaps & BGFX_CAPS_VIDEO_CODEC_BIT_8) - && 0 != (codecCaps & BGFX_CAPS_VIDEO_CODEC_CHROMA_420) - ; m_format = 0 != (caps->formats[bgfx::TextureFormat::RGBA8] & BGFX_CAPS_FORMAT_TEXTURE_VIDEO_DECODE_DST) ? bgfx::TextureFormat::RGBA8 : bgfx::TextureFormat::BGRA8 ; + const bool codecAvailable = 0 != (caps->supported & BGFX_CAPS_VIDEO_DECODE); const bool targetSupported = 0 != (caps->formats[m_format] & BGFX_CAPS_FORMAT_TEXTURE_VIDEO_DECODE_DST); const bool codecValid = bgfx::isVideoCodecValid( m_impl->codec @@ -685,6 +2643,10 @@ bool VideoPlayer::open(const char* _filePath) const uint32_t psSize = uint32_t(m_impl->parameterSets.size() ); const uint32_t totalSize = uint32_t(sizeof(bgfx::VideoDecoderInit) ) + psSize; + const uint64_t durationUs = bx::max(m_impl->frames.back().ptsUs, 1); + const uint64_t bytesPerSec = m_impl->totalBytes * 1000000ull / durationUs; + const uint32_t cachedAuBytes = uint32_t(bx::clamp(bytesPerSec * kCacheWindowUs / 1000000ull, kMinCachedAuBytes, UINT32_MAX) ); + const bgfx::Memory* mem = bgfx::alloc(totalSize); bgfx::VideoDecoderInit* decInit = (bgfx::VideoDecoderInit*)mem->data; @@ -694,7 +2656,7 @@ bool VideoPlayer::open(const char* _filePath) .codec = m_impl->codec, .parameterSets = mem->data + sizeof(bgfx::VideoDecoderInit), .parameterSetsSize = psSize, - .cachedAuBytes = 0, + .cachedAuBytes = cachedAuBytes, .flags = 0, }; @@ -850,17 +2812,48 @@ uint32_t VideoPlayer::findFrameAtOrBefore(uint64_t _ptsUs) const void VideoPlayer::submitDecodeFrame(uint32_t _index, bool _setPosition) { - const VideoPlayer::Impl::Frame& f = m_impl->frames[_index]; + const Impl::Frame& f = m_impl->frames[_index]; - lsmash_sample_t* s = lsmash_get_sample_from_media_timeline( - m_impl->root - , m_impl->trackId - , _index + 1 - ); + stl::vector& scratch = m_impl->scratch; + scratch.clear(); - if (NULL == s) + if (Impl::Mp4 == m_impl->container) { - return; + lsmash_sample_t* s = lsmash_get_sample_from_media_timeline( + m_impl->root + , m_impl->trackId + , _index + 1 + ); + + if (NULL == s) + { + return; + } + + if (bgfx::VideoCodec::AV1 == m_impl->codec) + { + scratch.insert(scratch.end(), s->data, s->data + s->length); + } + else + { + avccToAnnexB(scratch, s->data, s->length, m_impl->lengthSize); + } + + lsmash_delete_sample(s); + } + else + { + scratch.resize(f.size); + + bx::Error err; + bx::seek(&m_impl->reader, int64_t(f.offset), bx::Whence::Begin); + const int32_t read = bx::read(&m_impl->reader, scratch.data(), int32_t(f.size), &err); + + if (!err.isOk() + || uint32_t(read) != f.size) + { + return; + } } struct Payload @@ -869,9 +2862,6 @@ void VideoPlayer::submitDecodeFrame(uint32_t _index, bool _setPosition) bgfx::VideoDecoderAu au; }; - stl::vector& scratch = m_impl->annexBScratch; - scratch.clear(); - avccToAnnexB(scratch, s->data, s->length, m_impl->lengthSize); const uint32_t bitstreamSize = uint32_t(scratch.size() ); const uint32_t totalSize = uint32_t(sizeof(Payload) ) + bitstreamSize; @@ -909,8 +2899,6 @@ void VideoPlayer::submitDecodeFrame(uint32_t _index, bool _setPosition) , m_height , mem ); - - lsmash_delete_sample(s); } void VideoPlayer::submitAccessUnitsUpTo(int64_t _untilPtsUs) diff --git a/tools/texturev/video_player.h b/tools/texturev/video_player.h index 2cd282af2..cd43c0d8d 100644 --- a/tools/texturev/video_player.h +++ b/tools/texturev/video_player.h @@ -7,7 +7,7 @@ #define TEXTUREV_VIDEO_PLAYER_H_HEADER_GUARD #include -#include +#include class VideoPlayer { @@ -15,7 +15,7 @@ public: VideoPlayer(); ~VideoPlayer(); - static bool isVideoExt(const bx::StringView& _ext); + static bool isVideoFile(const bx::FilePath& _filePath); bool open(const char* _filePath); void close();