From 9b006616ed04bc5bc5bc077b6892541a7063b15b Mon Sep 17 00:00:00 2001
From: jeffqjiangNew <142832361+jeffqjiangNew@users.noreply.github.com>
Date: Thu, 15 Feb 2024 08:42:51 -0500
Subject: [PATCH] * rocDecode/AVC: Added SEI message extraction support. Also
merged common AVC and HEVC SEI support code to super class. (#250)
---
src/parser/avc_parser.cpp | 30 +++++++++-
src/parser/avc_parser.h | 4 ++
src/parser/hevc_defines.h | 2 -
src/parser/hevc_parser.cpp | 103 ++++++--------------------------
src/parser/hevc_parser.h | 29 ++++-----
src/parser/roc_video_parser.cpp | 66 ++++++++++++++++++++
src/parser/roc_video_parser.h | 11 +++-
7 files changed, 139 insertions(+), 106 deletions(-)
diff --git a/src/parser/avc_parser.cpp b/src/parser/avc_parser.cpp
index 502f5d1b98..fd7213fdbf 100644
--- a/src/parser/avc_parser.cpp
+++ b/src/parser/avc_parser.cpp
@@ -69,6 +69,11 @@ rocDecStatus AvcVideoParser::ParseVideoData(RocdecSourceDataPacket *p_data) {
new_sps_activated_ = false;
}
+ // Whenever new sei message found
+ if (pfn_get_sei_message_cb_ && sei_message_count_ > 0) {
+ SendSeiMsgPayload();
+ }
+
// Decode the picture
if (SendPicForDecode() != PARSER_OK) {
ERR(STR("Failed to decode!"));
@@ -209,7 +214,20 @@ ParserResult AvcVideoParser::ParsePictureData(const uint8_t *p_stream, uint32_t
case kAvcNalTypeSEI_Info: {
if (pfn_get_sei_message_cb_) {
- // Todo
+ int sei_ebsp_size = nal_unit_size_ - 4; // copy the entire NAL unit
+ if (sei_rbsp_buf_) {
+ if (sei_ebsp_size > sei_rbsp_buf_size_) {
+ delete [] sei_rbsp_buf_;
+ sei_rbsp_buf_ = new uint8_t [sei_ebsp_size];
+ sei_rbsp_buf_size_ = sei_ebsp_size;
+ }
+ } else {
+ sei_rbsp_buf_size_ = sei_ebsp_size > INIT_SEI_PAYLOAD_BUF_SIZE ? sei_ebsp_size : INIT_SEI_PAYLOAD_BUF_SIZE;
+ sei_rbsp_buf_ = new uint8_t [sei_rbsp_buf_size_];
+ }
+ memcpy(sei_rbsp_buf_, (pic_data_buffer_ptr_ + curr_start_code_offset_ + 4), sei_ebsp_size);
+ rbsp_size_ = EbspToRbsp(sei_rbsp_buf_, 0, sei_ebsp_size);
+ ParseSeiMessage(sei_rbsp_buf_, rbsp_size_);
}
break;
}
@@ -349,6 +367,16 @@ ParserResult AvcVideoParser::NotifyNewSps(AvcSeqParameterSet *p_sps) {
}
}
+void AvcVideoParser::SendSeiMsgPayload() {
+ sei_message_info_params_.sei_message_count = sei_message_count_;
+ sei_message_info_params_.sei_message = sei_message_list_.data();
+ sei_message_info_params_.sei_data = (void*)sei_payload_buf_;
+ sei_message_info_params_.picIdx = curr_pic_.pic_idx;
+
+ // callback function with RocdecSeiMessageInfo params filled out
+ if (pfn_get_sei_message_cb_) pfn_get_sei_message_cb_(parser_params_.user_data, &sei_message_info_params_);
+}
+
ParserResult AvcVideoParser::SendPicForDecode() {
int i, j;
AvcSeqParameterSet *p_sps = &sps_list_[active_sps_id_];
diff --git a/src/parser/avc_parser.h b/src/parser/avc_parser.h
index a0b82ad225..45ae8208b1 100644
--- a/src/parser/avc_parser.h
+++ b/src/parser/avc_parser.h
@@ -144,6 +144,10 @@ protected:
*/
ParserResult OutputDecodedPictures();
+ /*! \brief Callback function to send parsed SEI playload to decoder.
+ */
+ void SendSeiMsgPayload();
+
/*! \brief Function to parse one picture bit stream received from the demuxer.
* \param [in] p_stream A pointer of uint8_t for the input stream to be parsed
* \param [in] pic_data_size Size of the input stream
diff --git a/src/parser/hevc_defines.h b/src/parser/hevc_defines.h
index 01b07c11d9..91edf90f29 100644
--- a/src/parser/hevc_defines.h
+++ b/src/parser/hevc_defines.h
@@ -34,8 +34,6 @@ THE SOFTWARE.
#define HEVC_MAX_NUM_REF_PICS 16
// 7.4.7.1. (num_tile_columns_minus1 + 1) * PicHeightInCtbsY − 1. Max tile columns = 20 (A.4.2). Pic height in 16x16 CTB of 8K = 270.
#define MAX_ENTRY_POINT_OFFSETS 20 * 270
-#define INIT_SEI_MESSAGE_COUNT 16 // initial SEI message count
-#define INIT_SEI_PAYLOAD_BUF_SIZE 1024 * 1024 // initial SEI payload buffer size, 1 MB
/*! \brief Enumerator for the NAL Unit types - ISO-IEC 14496-15-2004.pdf, page 14, table 1 " NAL unit types in elementary streams
*/
diff --git a/src/parser/hevc_parser.cpp b/src/parser/hevc_parser.cpp
index 2cebc5d61a..5376525099 100644
--- a/src/parser/hevc_parser.cpp
+++ b/src/parser/hevc_parser.cpp
@@ -48,31 +48,34 @@ HevcVideoParser::HevcVideoParser() {
slice_info_list_.assign(INIT_SLICE_LIST_NUM, {0});
slice_param_list_.assign(INIT_SLICE_LIST_NUM, {0});
- sei_rbsp_buf_ = nullptr;
- sei_rbsp_buf_size_ = 0;
- sei_payload_buf_ = nullptr;
- sei_payload_buf_size_ = 0;
- sei_message_list_.assign(INIT_SEI_MESSAGE_COUNT, {0});
-
memset(&curr_pic_info_, 0, sizeof(HevcPicInfo));
InitDpb();
}
+HevcVideoParser::~HevcVideoParser() {
+ if (m_vps_) {
+ delete [] m_vps_;
+ }
+ if (m_sps_) {
+ delete [] m_sps_;
+ }
+ if (m_pps_) {
+ delete [] m_pps_;
+ }
+ if (m_sh_copy_) {
+ delete m_sh_copy_;
+ }
+}
+
rocDecStatus HevcVideoParser::Initialize(RocdecParserParams *p_params) {
return RocVideoParser::Initialize(p_params);
}
-/**
- * @brief function to uninitialize hevc parser
- *
- * @return rocDecStatus
- */
rocDecStatus HevcVideoParser::UnInitialize() {
//todo:: do any uninitialization here
return ROCDEC_SUCCESS;
}
-
rocDecStatus HevcVideoParser::ParseVideoData(RocdecSourceDataPacket *p_data) {
if (p_data->payload && p_data->payload_size) {
// Clear DPB output/display buffer number
@@ -94,7 +97,7 @@ rocDecStatus HevcVideoParser::ParseVideoData(RocdecSourceDataPacket *p_data) {
// Whenever new sei message found
if (pfn_get_sei_message_cb_ && sei_message_count_ > 0) {
- FillSeiMessageCallbackFn();
+ SendSeiMsgPayload();
}
// Decode the picture
@@ -125,27 +128,6 @@ rocDecStatus HevcVideoParser::ParseVideoData(RocdecSourceDataPacket *p_data) {
return ROCDEC_SUCCESS;
}
-HevcVideoParser::~HevcVideoParser() {
- if (m_vps_) {
- delete [] m_vps_;
- }
- if (m_sps_) {
- delete [] m_sps_;
- }
- if (m_pps_) {
- delete [] m_pps_;
- }
- if (m_sh_copy_) {
- delete m_sh_copy_;
- }
- if (sei_rbsp_buf_) {
- delete [] sei_rbsp_buf_;
- }
- if (sei_payload_buf_) {
- delete [] sei_payload_buf_;
- }
-}
-
int HevcVideoParser::FillSeqCallbackFn(HevcSeqParamSet* sps_data) {
video_format_params_.codec = rocDecVideoCodec_HEVC;
video_format_params_.frame_rate.numerator = frame_rate_.numerator;
@@ -247,7 +229,7 @@ int HevcVideoParser::FillSeqCallbackFn(HevcSeqParamSet* sps_data) {
}
}
-void HevcVideoParser::FillSeiMessageCallbackFn() {
+void HevcVideoParser::SendSeiMsgPayload() {
sei_message_info_params_.sei_message_count = sei_message_count_;
sei_message_info_params_.sei_message = sei_message_list_.data();
sei_message_info_params_.sei_data = (void*)sei_payload_buf_;
@@ -1846,57 +1828,6 @@ bool HevcVideoParser::ParseSliceHeader(uint8_t *nalu, size_t size, HevcSliceSegH
return false;
}
-void HevcVideoParser::ParseSeiMessage(uint8_t *nalu, size_t size) {
- int offset = 0; // byte offset
- int payload_type;
- int payload_size;
-
- do {
- payload_type = 0;
- while (nalu[offset] == 0xFF) {
- payload_type += 255; // ff_byte
- offset++;
- }
- payload_type += nalu[offset]; // last_payload_type_byte
- offset++;
-
- payload_size = 0;
- while (nalu[offset] == 0xFF) {
- payload_size += 255; // ff_byte
- offset++;
- }
- payload_size += nalu[offset]; // last_payload_size_byte
- offset++;
-
- // We start with INIT_SEI_MESSAGE_COUNT. Should be enough for normal use cases. If not, resize.
- if((sei_message_count_ + 1) > sei_message_list_.size()) {
- sei_message_list_.resize((sei_message_count_ + 1));
- }
- sei_message_list_[sei_message_count_].sei_message_type = payload_type;
- sei_message_list_[sei_message_count_].sei_message_size = payload_size;
-
- if (sei_payload_buf_) {
- if ((payload_size + sei_payload_size_) > sei_payload_buf_size_) {
- uint8_t *tmp_ptr = new uint8_t [payload_size + sei_payload_size_];
- memcpy(tmp_ptr, sei_payload_buf_, sei_payload_size_); // save the existing payload
- delete [] sei_payload_buf_;
- sei_payload_buf_ = tmp_ptr;
- }
- } else {
- // First payload, sei_payload_size_ is 0.
- sei_payload_buf_size_ = payload_size > INIT_SEI_PAYLOAD_BUF_SIZE ? payload_size : INIT_SEI_PAYLOAD_BUF_SIZE;
- sei_payload_buf_ = new uint8_t [sei_payload_buf_size_];
- }
- // Append the current payload to sei_payload_buf_
- memcpy(sei_payload_buf_ + sei_payload_size_, nalu + offset, payload_size);
-
- sei_payload_size_ += payload_size;
- sei_message_count_++;
-
- offset += payload_size;
- } while (offset < size && nalu[offset] != 0x80);
-}
-
bool HevcVideoParser::IsIdrPic(HevcNalUnitHeader *nal_header_ptr) {
return (nal_header_ptr->nal_unit_type == NAL_UNIT_CODED_SLICE_IDR_W_RADL || nal_header_ptr->nal_unit_type == NAL_UNIT_CODED_SLICE_IDR_N_LP);
}
diff --git a/src/parser/hevc_parser.h b/src/parser/hevc_parser.h
index 51162f6757..679854eb7c 100644
--- a/src/parser/hevc_parser.h
+++ b/src/parser/hevc_parser.h
@@ -42,6 +42,11 @@ public:
*/
HevcVideoParser();
+ /**
+ * @brief HEVCParser object destructor
+ */
+ virtual ~HevcVideoParser();
+
/*! \brief Function to Initialize the parser
* \param [in] p_params Input of RocdecParserParams with codec type to initialize parser.
* \return rocDecStatus Returns success on completion, else error code for failure
@@ -61,11 +66,6 @@ public:
*/
virtual rocDecStatus UnInitialize(); // derived method :: nothing to do for this
- /**
- * @brief HEVCParser object destructor
- */
- virtual ~HevcVideoParser();
-
protected:
/*! \brief Inline function to Parse the NAL Unit Header
*
@@ -277,13 +277,6 @@ protected:
*/
bool ParseSliceHeader(uint8_t *nalu, size_t size, HevcSliceSegHeader *p_slice_header);
- /*! \brief Function to parse Sei Message Info
- * \param [in] nalu A pointer of uint8_t for the input stream to be parsed
- * \param [in] size Size of the input stream
- * \return No return value
- */
- void ParseSeiMessage(uint8_t *nalu, size_t size);
-
/*! \brief Function to calculate the picture order count of the current picture. Once per picutre. (8.3.1)
*/
void CalculateCurrPoc();
@@ -343,16 +336,20 @@ protected:
#endif // DBGINFO
private:
- // functions to fill structures for callback functions
+ /*! \brief Callback function to notify decoder about new SPS.
+ */
int FillSeqCallbackFn(HevcSeqParamSet* sps_data);
- void FillSeiMessageCallbackFn();
- /*! \brief Function to fill the decode parameters and call back decoder to decode a picture
+ /*! \brief Callback function to send parsed SEI playload to decoder.
+ */
+ void SendSeiMsgPayload();
+
+ /*! \brief Callback function to fill the decode parameters and call decoder to decode a picture
* \return Return code in ParserResult form
*/
int SendPicForDecode();
- /*! \brief Function to output decoded pictures from DPB for post-processing.
+ /*! \brief Callback function to output decoded pictures from DPB for post-processing.
* \return Return code in ParserResult form
*/
int OutputDecodedPictures();
diff --git a/src/parser/roc_video_parser.cpp b/src/parser/roc_video_parser.cpp
index ccaef3e23c..41c8ca4427 100644
--- a/src/parser/roc_video_parser.cpp
+++ b/src/parser/roc_video_parser.cpp
@@ -29,6 +29,21 @@ RocVideoParser::RocVideoParser() {
new_sps_activated_ = false;
frame_rate_.numerator = 0;
frame_rate_.denominator = 0;
+
+ sei_rbsp_buf_ = nullptr;
+ sei_rbsp_buf_size_ = 0;
+ sei_payload_buf_ = nullptr;
+ sei_payload_buf_size_ = 0;
+ sei_message_list_.assign(INIT_SEI_MESSAGE_COUNT, {0});
+}
+
+RocVideoParser::~RocVideoParser() {
+ if (sei_rbsp_buf_) {
+ delete [] sei_rbsp_buf_;
+ }
+ if (sei_payload_buf_) {
+ delete [] sei_payload_buf_;
+ }
}
/**
@@ -132,3 +147,54 @@ size_t RocVideoParser::EbspToRbsp(uint8_t *streamBuffer,size_t begin_bytepos, si
}
return end_bytepos - begin_bytepos + reduce_count;
}
+
+void RocVideoParser::ParseSeiMessage(uint8_t *nalu, size_t size) {
+ int offset = 0; // byte offset
+ int payload_type;
+ int payload_size;
+
+ do {
+ payload_type = 0;
+ while (nalu[offset] == 0xFF) {
+ payload_type += 255; // ff_byte
+ offset++;
+ }
+ payload_type += nalu[offset]; // last_payload_type_byte
+ offset++;
+
+ payload_size = 0;
+ while (nalu[offset] == 0xFF) {
+ payload_size += 255; // ff_byte
+ offset++;
+ }
+ payload_size += nalu[offset]; // last_payload_size_byte
+ offset++;
+
+ // We start with INIT_SEI_MESSAGE_COUNT. Should be enough for normal use cases. If not, resize.
+ if((sei_message_count_ + 1) > sei_message_list_.size()) {
+ sei_message_list_.resize((sei_message_count_ + 1));
+ }
+ sei_message_list_[sei_message_count_].sei_message_type = payload_type;
+ sei_message_list_[sei_message_count_].sei_message_size = payload_size;
+
+ if (sei_payload_buf_) {
+ if ((payload_size + sei_payload_size_) > sei_payload_buf_size_) {
+ uint8_t *tmp_ptr = new uint8_t [payload_size + sei_payload_size_];
+ memcpy(tmp_ptr, sei_payload_buf_, sei_payload_size_); // save the existing payload
+ delete [] sei_payload_buf_;
+ sei_payload_buf_ = tmp_ptr;
+ }
+ } else {
+ // First payload, sei_payload_size_ is 0.
+ sei_payload_buf_size_ = payload_size > INIT_SEI_PAYLOAD_BUF_SIZE ? payload_size : INIT_SEI_PAYLOAD_BUF_SIZE;
+ sei_payload_buf_ = new uint8_t [sei_payload_buf_size_];
+ }
+ // Append the current payload to sei_payload_buf_
+ memcpy(sei_payload_buf_ + sei_payload_size_, nalu + offset, payload_size);
+
+ sei_payload_size_ += payload_size;
+ sei_message_count_++;
+
+ offset += payload_size;
+ } while (offset < size && nalu[offset] != 0x80);
+}
\ No newline at end of file
diff --git a/src/parser/roc_video_parser.h b/src/parser/roc_video_parser.h
index f98acde6c4..62967d5bda 100644
--- a/src/parser/roc_video_parser.h
+++ b/src/parser/roc_video_parser.h
@@ -76,6 +76,8 @@ typedef struct {
#define ZEROBYTES_SHORTSTARTCODE 2 //indicates the number of zero bytes in the short start-code prefix
#define RBSP_BUF_SIZE 1024 // enough to parse any parameter sets or slice headers
#define INIT_SLICE_LIST_NUM 16 // initial slice information/parameter struct list size
+#define INIT_SEI_MESSAGE_COUNT 16 // initial SEI message count
+#define INIT_SEI_PAYLOAD_BUF_SIZE 1024 * 1024 // initial SEI payload buffer size, 1 MB
/**
* @brief Base class for video parsing
@@ -84,8 +86,8 @@ typedef struct {
class RocVideoParser {
public:
RocVideoParser(); // default constructor
+ virtual ~RocVideoParser();
RocVideoParser(RocdecParserParams *pParams) : parser_params_(*pParams) {};
- virtual ~RocVideoParser() = default ;
virtual void SetParserParams(RocdecParserParams *pParams) { parser_params_ = *pParams; };
RocdecParserParams *GetParserParams() {return &parser_params_;};
virtual rocDecStatus Initialize(RocdecParserParams *pParams);
@@ -152,6 +154,13 @@ protected:
* \return Returns the size of the converted buffer in size_t
*/
size_t EbspToRbsp(uint8_t *stream_buffer, size_t begin_bytepos, size_t end_bytepos);
+
+ /*! \brief Function to parse Sei Message Info
+ * \param [in] nalu A pointer of uint8_t for the input stream to be parsed
+ * \param [in] size Size of the input stream
+ * \return No return value
+ */
+ void ParseSeiMessage(uint8_t *nalu, size_t size);
};
// helpers