From d55324bcf9278135121d4030fb351f91545a2759 Mon Sep 17 00:00:00 2001 From: Rajy Rawther Date: Tue, 19 Dec 2023 05:49:17 -0800 Subject: [PATCH] Add mechanism to flush last frames during reconfigure (#142) * add mechanism to flush last frames during reconfigure * reconfig mode updated support * addressed review comments * addressed review comments * change condition according to review comment [ROCm/rocdecode commit: a5211189f5a7dba4876d3251a96d2239a0f72e7f] --- projects/rocdecode/samples/common.h | 66 ++++++++++++++ .../samples/videoDecode/CMakeLists.txt | 2 +- .../samples/videoDecode/videodecode.cpp | 23 ++++- .../videoDecodeMultiFiles/CMakeLists.txt | 2 +- .../videodecodemultifiles.cpp | 19 +++- .../utils/rocvideodecode/roc_video_dec.cpp | 91 ++++++++++++++++++- .../utils/rocvideodecode/roc_video_dec.h | 37 +++++++- 7 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 projects/rocdecode/samples/common.h diff --git a/projects/rocdecode/samples/common.h b/projects/rocdecode/samples/common.h new file mode 100644 index 0000000000..225fcc215d --- /dev/null +++ b/projects/rocdecode/samples/common.h @@ -0,0 +1,66 @@ +/* +Copyright (c) 2023 - 2023 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#pragma once + +#include "roc_video_dec.h" + +typedef enum ReconfigFlushMode_enum { + RECONFIG_FLUSH_MODE_DUMP_TO_FILE = 0, /**< The remaining frames will be dumped to file in this mode */ + //TODO;; Add new flush modes here : defined by the application +} ReconfigFlushMode; + +// this struct is used by videodecode and videodecodeMultiFiles to dump last frames to file +typedef struct ReconfigDumpFileStruct_t { + bool b_dump_frames_to_file; + std::string output_file_name; +} ReconfigDumpFileStruct; + + +// callback function to flush last frames and save it to file when reconfigure happens +int ReconfigureFlushCallback(void *p_viddec_obj, uint32_t flush_mode, void *p_user_struct) { + int n_frames_flushed = 0; + if ((p_viddec_obj == nullptr) || (p_user_struct == nullptr)) return n_frames_flushed; + + RocVideoDecoder *viddec = static_cast (p_viddec_obj); + OutputSurfaceInfo *surf_info; + if (!viddec->GetOutputSurfaceInfo(&surf_info)) { + std::cerr << "Error: Failed to get Output Surface Info!" << std::endl; + return n_frames_flushed; + } + if (flush_mode == ReconfigFlushMode::RECONFIG_FLUSH_MODE_DUMP_TO_FILE) { + ReconfigDumpFileStruct *p_dump_file_struct = static_cast(p_user_struct); + uint8_t *pframe = nullptr; + int64_t pts; + while ((pframe = viddec->GetFrame(&pts))) { + if (p_dump_file_struct->b_dump_frames_to_file) { + viddec->SaveFrameToFile(p_dump_file_struct->output_file_name, pframe, surf_info); + } + // release and flush frame + viddec->ReleaseFrame(pts, true); + n_frames_flushed ++; + } + } else { + //todo:: handle other cases + } + return n_frames_flushed; +} diff --git a/projects/rocdecode/samples/videoDecode/CMakeLists.txt b/projects/rocdecode/samples/videoDecode/CMakeLists.txt index 955bf53509..bbdb0a4ea0 100644 --- a/projects/rocdecode/samples/videoDecode/CMakeLists.txt +++ b/projects/rocdecode/samples/videoDecode/CMakeLists.txt @@ -75,7 +75,7 @@ if(HIP_FOUND AND FFMPEG_FOUND AND ROCDECODE_FOUND) ${AVFORMAT_INCLUDE_DIR}) set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${FFMPEG_LIBRARIES}) # rocDecode - include_directories (${ROCDECODE_INCLUDE_DIR}) + include_directories (${ROCDECODE_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/..) set(LINK_LIBRARY_LIST ${LINK_LIBRARY_LIST} ${ROCDECODE_LIBRARY}) list(APPEND SOURCES ${PROJECT_SOURCE_DIR} videodecode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/rocvideodecode/roc_video_dec.cpp) diff --git a/projects/rocdecode/samples/videoDecode/videodecode.cpp b/projects/rocdecode/samples/videoDecode/videodecode.cpp index 63ee431dc6..12664f1e33 100644 --- a/projects/rocdecode/samples/videoDecode/videodecode.cpp +++ b/projects/rocdecode/samples/videoDecode/videodecode.cpp @@ -38,6 +38,7 @@ THE SOFTWARE. #endif #include "video_demuxer.h" #include "roc_video_dec.h" +#include "common.h" void ShowHelpAndExit(const char *option = NULL) { std::cout << "Options:" << std::endl @@ -64,9 +65,13 @@ int main(int argc, char **argv) { bool b_extract_sei_messages = false; bool b_generate_md5 = false; bool b_md5_check = false; + bool b_flush_frames_during_reconfig = true; Rect crop_rect = {}; Rect *p_crop_rect = nullptr; - OutputSurfaceMemoryType mem_type = OUT_SURFACE_MEM_DEV_INTERNAL; // set to internal + OutputSurfaceMemoryType mem_type = OUT_SURFACE_MEM_DEV_INTERNAL; // set to internal + ReconfigParams reconfig_params = { 0 }; + ReconfigDumpFileStruct reconfig_user_struct = { 0 }; + // Parse command-line arguments if(argc <= 1) { ShowHelpAndExit(); @@ -145,8 +150,14 @@ int main(int argc, char **argv) { mem_type = static_cast(atoi(argv[i])); continue; } + if (!strcmp(argv[i], "flush")) { + b_flush_frames_during_reconfig = atoi(argv[i]) ? true : false; + continue; + } + ShowHelpAndExit(argv[i]); } + try { VideoDemuxer demuxer(input_file_path.c_str()); rocDecVideoCodec rocdec_codec_id = AVCodec2RocDecVideoCodec(demuxer.GetCodecID()); @@ -171,6 +182,12 @@ int main(int argc, char **argv) { OutputSurfaceInfo *surf_info; uint32_t width, height; double total_dec_time = 0; + // initialize reconfigure params: the following is configured to dump to output which is relevant for this sample + reconfig_params.p_fn_reconfigure_flush = ReconfigureFlushCallback; + reconfig_user_struct.b_dump_frames_to_file = dump_output_frames; + reconfig_user_struct.output_file_name = output_file_path; + reconfig_params.reconfig_flush_mode = RECONFIG_FLUSH_MODE_DUMP_TO_FILE; + reconfig_params.p_reconfig_user_struct = &reconfig_user_struct; if (b_generate_md5) { viddec.InitMd5(); @@ -178,6 +195,7 @@ int main(int argc, char **argv) { if (b_md5_check) { ref_md5_file.open(md5_file_path.c_str(), std::ios::in); } + if (dump_output_frames) viddec.SetReconfigParams(&reconfig_params); do { auto start_time = std::chrono::high_resolution_clock::now(); @@ -207,7 +225,8 @@ int main(int argc, char **argv) { } n_frame += n_frame_returned; } while (n_video_bytes); - + + n_frame += viddec.GetNumOfFlushedFrames(); std::cout << "info: Total frame decoded: " << n_frame << std::endl; if (!dump_output_frames) { std::cout << "info: avg decoding time per frame: " << total_dec_time / n_frame << " ms" < *multi_file_data, int &device_id, boo file_idx++; file_data.b_force_zero_latency = false; file_data.b_extract_sei_messages = false; + file_data.b_flush_last_frames = true; file_data.dump_output_frames = 0; file_data.crop_rect = {0, 0, 0, 0}; file_data.p_crop_rect = nullptr; @@ -137,6 +141,8 @@ void ParseCommandLine(std::deque *multi_file_data, int &device_id, boo file_data.b_force_zero_latency = atoi(value) ? true : false; } else if (!strcmp(param, "sei")) { file_data.b_extract_sei_messages = atoi(value) ? true : false; + } else if (!strcmp(param, "flush")) { + file_data.b_flush_last_frames = atoi(value) ? true : false; } else if (!strcmp(param, "crop")) { sscanf(value, "%d,%d,%d,%d", &file_data.crop_rect.l, &file_data.crop_rect.t, &file_data.crop_rect.r, &file_data.crop_rect.b); if ((file_data.crop_rect.r - file_data.crop_rect.l) % 2 == 1 || (file_data.crop_rect.b - file_data.crop_rect.t) % 2 == 1) { @@ -162,6 +168,8 @@ int main(int argc, char **argv) { ParseCommandLine (&multi_file_data, device_id, use_reconfigure, argc, argv); RocVideoDecoder *viddec = NULL; + ReconfigParams reconfig_params = { 0 }; + ReconfigDumpFileStruct reconfig_user_struct = { 0 }; try { while (!multi_file_data.empty()) { @@ -169,14 +177,22 @@ int main(int argc, char **argv) { multi_file_data.pop_front(); VideoDemuxer demuxer(file_data.in_file.c_str()); rocDecVideoCodec rocdec_codec_id = AVCodec2RocDecVideoCodec(demuxer.GetCodecID()); - + if (file_data.b_flush_last_frames && file_data.dump_output_frames) { + reconfig_params.p_fn_reconfigure_flush = ReconfigureFlushCallback; + reconfig_user_struct.b_dump_frames_to_file = file_data.dump_output_frames; + reconfig_user_struct.output_file_name = file_data.out_file; + reconfig_params.reconfig_flush_mode = RECONFIG_FLUSH_MODE_DUMP_TO_FILE; + reconfig_params.p_reconfig_user_struct = &reconfig_user_struct; + } if (use_reconfigure) { + if (!viddec) { viddec = new RocVideoDecoder(device_id, file_data.mem_type, rocdec_codec_id, file_data.b_force_zero_latency, file_data.p_crop_rect, file_data.b_extract_sei_messages); } } else { viddec = new RocVideoDecoder(device_id, file_data.mem_type, rocdec_codec_id, file_data.b_force_zero_latency, file_data.p_crop_rect, file_data.b_extract_sei_messages); } + if (viddec && file_data.b_flush_last_frames) viddec->SetReconfigParams(&reconfig_params); std::string device_name, gcn_arch_name; int pci_bus_id, pci_domain_id, pci_device_id; @@ -223,6 +239,7 @@ int main(int argc, char **argv) { n_frame += n_frame_returned; } while (n_video_bytes); + n_frame += viddec->GetNumOfFlushedFrames(); std::cout << "info: Total frame decoded: " << n_frame << std::endl; if (!file_data.dump_output_frames) { std::cout << "info: avg decoding time per frame (ms): " << total_dec_time / n_frame << std::endl; diff --git a/projects/rocdecode/utils/rocvideodecode/roc_video_dec.cpp b/projects/rocdecode/utils/rocvideodecode/roc_video_dec.cpp index fd30ae18f8..ba450730bc 100644 --- a/projects/rocdecode/utils/rocvideodecode/roc_video_dec.cpp +++ b/projects/rocdecode/utils/rocvideodecode/roc_video_dec.cpp @@ -24,8 +24,8 @@ THE SOFTWARE. RocVideoDecoder::RocVideoDecoder(int device_id, OutputSurfaceMemoryType out_mem_type, rocDecVideoCodec codec, bool force_zero_latency, const Rect *p_crop_rect, bool extract_user_sei_Message, int max_width, int max_height, uint32_t clk_rate) : - device_id_{device_id}, out_mem_type_(out_mem_type), codec_id_(codec), b_force_zero_latency_(force_zero_latency), b_extract_sei_message_(extract_user_sei_Message), - max_width_ (max_width), max_height_(max_height) { + device_id_{device_id}, out_mem_type_(out_mem_type), codec_id_(codec), b_force_zero_latency_(force_zero_latency), + b_extract_sei_message_(extract_user_sei_Message), max_width_ (max_width), max_height_(max_height) { if (!InitHIP(device_id_)) { THROW("Failed to initilize the HIP"); @@ -417,6 +417,24 @@ int RocVideoDecoder::HandleVideoSequence(RocdecVideoFormat *p_video_format) { return nDecodeSurface; } +/** + * @brief Function to set the Reconfig Params object + * + * @param p_reconfig_params: pointer to reconfig params struct + * @return true : success + * @return false : fail + */ +bool RocVideoDecoder::SetReconfigParams(ReconfigParams *p_reconfig_params) { + if (!p_reconfig_params) { + std::cout << "ERROR: Invalid reconfig struct passed! "<< std::endl; + return false; + } + //save it + p_reconfig_params_ = p_reconfig_params; + return true; +} + + /** * @brief function to reconfigure decoder if there is a change in sequence params. * @@ -453,6 +471,31 @@ int RocVideoDecoder::ReconfigureDecoder(RocdecVideoFormat *p_video_format) { } return 1; } + // flush and clear internal frame store to reconfigure + if (p_reconfig_params_ && p_reconfig_params_->p_fn_reconfigure_flush) + num_frames_flushed_during_reconfig_ = p_reconfig_params_->p_fn_reconfigure_flush(this, p_reconfig_params_->reconfig_flush_mode, + static_cast(p_reconfig_params_->p_reconfig_user_struct)); + // clear the existing output buffers of different size + // note that app lose the remaining frames in the vp_frames/vp_frames_q in case application didn't set p_fn_reconfigure_flush_ callback + if (out_mem_type_ == OUT_SURFACE_MEM_DEV_INTERNAL) { + ReleaseInternalFrames(); + } else { + std::lock_guard lock(mtx_vp_frame_); + while(!vp_frames_.empty()) { + DecFrameBuffer *p_frame = &vp_frames_.back(); + // pop decoded frame + vp_frames_.pop_back(); + if (p_frame->frame_ptr) { + if (out_mem_type_ == OUT_SURFACE_MEM_DEV_COPIED) { + hipError_t hip_status = hipFree(p_frame->frame_ptr); + if (hip_status != hipSuccess) std::cout << "ERROR: hipFree failed! (" << hip_status << ")" << std::endl; + } + else + delete [] (p_frame->frame_ptr); + } + } + } + decoded_frame_cnt_ = 0; //reset frame_count width_ = p_video_format->coded_width; height_ = p_video_format->coded_height; @@ -477,6 +520,9 @@ int RocVideoDecoder::ReconfigureDecoder(RocdecVideoFormat *p_video_format) { } else { surface_stride_ = reconfig_params.ulTargetWidth * byte_per_pixel_; } + chroma_height_ = static_cast(ceil(height_ * GetChromaHeightFactor(video_surface_format_))); + num_chroma_planes_ = GetChromaPlaneCount(video_surface_format_); + if (p_video_format->chroma_format == rocDecVideoChromaFormat_Monochrome) num_chroma_planes_ = 0; chroma_vstride_ = static_cast(std::ceil(surface_vstride_ * GetChromaHeightFactor(video_surface_format_))); // fill output_surface_info_ output_surface_info_.output_width = width_; @@ -504,6 +550,7 @@ int RocVideoDecoder::ReconfigureDecoder(RocdecVideoFormat *p_video_format) { } ROCDEC_API_CALL(rocDecReconfigureDecoder(roc_decoder_, &reconfig_params)); + input_video_info_str_.str(""); input_video_info_str_.clear(); input_video_info_str_ << "Input Video Resolution Changed:" << std::endl @@ -744,6 +791,8 @@ uint8_t* RocVideoDecoder::GetFrame(int64_t *pts) { return nullptr; } + + /** * @brief function to release frame after use by the application: Only used with "OUT_SURFACE_MEM_DEV_INTERNAL" * @@ -752,9 +801,19 @@ uint8_t* RocVideoDecoder::GetFrame(int64_t *pts) { * @return false - falied */ -bool RocVideoDecoder::ReleaseFrame(int64_t pTimestamp) { - if (out_mem_type_ != OUT_SURFACE_MEM_DEV_INTERNAL) - return true; // nothing to do +bool RocVideoDecoder::ReleaseFrame(int64_t pTimestamp, bool b_flushing) { + if (out_mem_type_ != OUT_SURFACE_MEM_DEV_INTERNAL) { + if (!b_flushing) // if not flushing the buffers are re-used, so keep them + return true; // nothing to do + else { + DecFrameBuffer *fb = &vp_frames_[0]; + if (pTimestamp != fb->pts) { + std::cerr << "Decoded Frame is released out of order" << std::endl; + return false; + } + vp_frames_.erase(vp_frames_.begin()); // get rid of the frames from the framestore + } + } // only needed when using internal mapped buffer if (!vp_frames_q_.empty()) { std::lock_guard lock(mtx_vp_frame_); @@ -772,6 +831,28 @@ bool RocVideoDecoder::ReleaseFrame(int64_t pTimestamp) { return true; } + +/** + * @brief function to release all internal frames and clear the q (used with reconfigure): Only used with "OUT_SURFACE_MEM_DEV_INTERNAL" + * + * @return true - success + * @return false - falied + */ +bool RocVideoDecoder::ReleaseInternalFrames() { + if (out_mem_type_ != OUT_SURFACE_MEM_DEV_INTERNAL) + return true; // nothing to do + // only needed when using internal mapped buffer + while (!vp_frames_q_.empty()) { + std::lock_guard lock(mtx_vp_frame_); + DecFrameBuffer *fb = &vp_frames_q_.front(); + ROCDEC_API_CALL(rocDecUnMapVideoFrame(roc_decoder_, fb->picture_index)); + // pop decoded frame + vp_frames_q_.pop(); + } + return true; +} + + void RocVideoDecoder::SaveFrameToFile(std::string output_file_name, void *surf_mem, OutputSurfaceInfo *surf_info) { uint8_t *hst_ptr = nullptr; uint64_t output_image_size = surf_info->output_surface_size_in_bytes; diff --git a/projects/rocdecode/utils/rocvideodecode/roc_video_dec.h b/projects/rocdecode/utils/rocvideodecode/roc_video_dec.h index c52ca4fd9c..5ec0b726ad 100644 --- a/projects/rocdecode/utils/rocvideodecode/roc_video_dec.h +++ b/projects/rocdecode/utils/rocvideodecode/roc_video_dec.h @@ -42,6 +42,7 @@ extern "C" { #include "rocparser.h" #define MAX_FRAME_NUM 16 +typedef int (ROCDECAPI *PFNRECONFIGUEFLUSHCALLBACK)(void *, uint32_t, void *); typedef enum SeiH264HevcPayloadType_enum { SEI_TYPE_TIME_CODE = 136, @@ -134,6 +135,12 @@ typedef struct OutputSurfaceInfoType { OutputSurfaceMemoryType mem_type; /**< Output mem_type of the surface*/ } OutputSurfaceInfo; +typedef struct ReconfigParams_t { + PFNRECONFIGUEFLUSHCALLBACK p_fn_reconfigure_flush; + void *p_reconfig_user_struct; + uint32_t reconfig_flush_mode; +} ReconfigParams; + class RocVideoDecoder { public: /** @@ -233,6 +240,15 @@ class RocVideoDecoder { * @return false */ bool GetOutputSurfaceInfo(OutputSurfaceInfo **surface_info); + + /** + * @brief Function to set the Reconfig Params object + * + * @param p_reconfig_params: pointer to reconfig params struct + * @return true : success + * @return false : fail + */ + bool SetReconfigParams(ReconfigParams *p_reconfig_params); /** * @brief this function decodes a frame and returns the number of frames avalable for display * @@ -253,10 +269,11 @@ class RocVideoDecoder { * @brief function to release frame after use by the application: Only used with "OUT_SURFACE_MEM_DEV_INTERNAL" * * @param pTimestamp - timestamp of the frame to be released (unmapped) + * @param b_flushing - true when flushing * @return true - success * @return false - falied */ - bool ReleaseFrame(int64_t pTimestamp); + bool ReleaseFrame(int64_t pTimestamp, bool b_flushing = false); /** * @brief utility function to save image to a file @@ -307,6 +324,12 @@ class RocVideoDecoder { * @param [out] digest Pointer to the 16 byte message digest */ void FinalizeMd5(uint8_t **digest); + /** + * @brief Get the Num Of Flushed Frames from video decoder object + * + * @return int32_t + */ + int32_t GetNumOfFlushedFrames() { return num_frames_flushed_during_reconfig_;} private: int decoder_session_id_; // Decoder session identifier. Used to gather session level stats. @@ -356,12 +379,21 @@ class RocVideoDecoder { * @brief This function reconfigure decoder if there is a change in sequence params. */ int ReconfigureDecoder(RocdecVideoFormat *p_video_format); + + /** + * @brief function to release all internal frames and clear the vp_frames_q_ (used with reconfigure): Only used with "OUT_SURFACE_MEM_DEV_INTERNAL" + * + * @return true - success + * @return false - falied + */ + bool ReleaseInternalFrames(); /** * @brief Function to Initialize GPU-HIP * */ bool InitHIP(int device_id); + int num_devices_; int device_id_; RocdecVideoParser rocdec_parser_ = nullptr; @@ -369,7 +401,8 @@ class RocVideoDecoder { OutputSurfaceMemoryType out_mem_type_ = OUT_SURFACE_MEM_DEV_INTERNAL; bool b_extract_sei_message_ = false; bool b_force_zero_latency_ = false; - //bool b_device_frame_pitched_ = true; + ReconfigParams *p_reconfig_params_ = nullptr; + int32_t num_frames_flushed_during_reconfig_ = 0; hipDeviceProp_t hip_dev_prop_; hipStream_t hip_stream_; rocDecVideoCodec codec_id_ = rocDecVideoCodec_NumCodecs;