diff --git a/projects/clr/hipamd/src/hip_fatbin.cpp b/projects/clr/hipamd/src/hip_fatbin.cpp index a75fd82be6..57c0c718c2 100644 --- a/projects/clr/hipamd/src/hip_fatbin.cpp +++ b/projects/clr/hipamd/src/hip_fatbin.cpp @@ -24,6 +24,7 @@ THE SOFTWARE. #include #include "hip_code_object.hpp" +#include "hip_platform.hpp" namespace hip { @@ -50,28 +51,50 @@ FatBinaryInfo::FatBinaryInfo(const char* fname, const void* image) : fdesc_(amd: FatBinaryInfo::~FatBinaryInfo() { - for (auto* fbd: fatbin_dev_info_) { - if (fbd != nullptr) { - delete fbd; + if (!HIP_USE_RUNTIME_UNBUNDLER) { + // Using COMGR Unbundler + if (ufd_ && ufd_->fdesc_ > 0) { + // Check for ufd_ != nullptr, since sometimes, we never create unique_file_desc. + if (ufd_->fsize_ && image_mapped_ + && !amd::Os::MemoryUnmapFile(image_, ufd_->fsize_)) { + LogPrintfError("Cannot unmap file for fdesc: %d fsize: %d", ufd_->fdesc_, ufd_->fsize_); + assert(false); + } + if (!PlatformState::instance().CloseUniqueFileHandle(ufd_)) { + LogPrintfError("Cannot close file for fdesc: %d", ufd_->fdesc_); + assert(false); + } } - } - if (fdesc_ > 0) { - if (fsize_ - && (HIP_USE_RUNTIME_UNBUNDLER || image_mapped_) - && !amd::Os::MemoryUnmapFile(image_, fsize_)) { - guarantee(false, "Cannot unmap file for fdesc: %d fsize: %d \n", fdesc_, fsize_); - } - if (!amd::Os::CloseFileHandle(fdesc_)) { - guarantee(false, "Cannot close file for fdesc: %d \n", fdesc_); - } - } + fname_ = std::string(); + fdesc_ = amd::Os::FDescInit(); + fsize_ = 0; + image_ = nullptr; + uri_ = std::string(); - fname_ = std::string(); - fdesc_ = amd::Os::FDescInit(); - fsize_ = 0; - image_ = nullptr; - uri_ = std::string(); + if (0 == PlatformState::instance().UfdMapSize()) { + LogError("All Unique FDs are closed"); + } + + } else { + // Using Runtime Unbundler + if (fdesc_ > 0) { + if (fsize_ && !amd::Os::MemoryUnmapFile(image_, fsize_)) { + LogPrintfError("Cannot unmap file for fdesc: %d fsize: %d", fdesc_, fsize_); + assert(false); + } + if (!amd::Os::CloseFileHandle(fdesc_)) { + LogPrintfError("Cannot close file for fdesc: %d", fdesc_); + assert(false); + } + } + + fname_ = std::string(); + fdesc_ = amd::Os::FDescInit(); + fsize_ = 0; + image_ = nullptr; + uri_ = std::string(); + } } void ListAllDeviceWithNoCOFromBundle(const std::unordered_map 0) { // Get File Handle & size of the file. - if (!amd::Os::GetFileHandle(fname_.c_str(), &fdesc_, &fsize_)) + ufd_ = PlatformState::instance().GetUniqueFileHandle(fname_.c_str()); + if (ufd_ == nullptr) { return hipErrorFileNotFound; + } // If the file name exists but the file size is 0, the something wrong with the file or its path - if (fsize_ == 0) + if (ufd_->fsize_ == 0) { return hipErrorInvalidValue; + } // If image_ is nullptr, then file path is passed via hipMod* APIs, so map the file. if (image_ == nullptr) { - if(!amd::Os::MemoryMapFileDesc(fdesc_, fsize_, foffset_, &image_)) { + if(!amd::Os::MemoryMapFileDesc(ufd_->fdesc_, ufd_->fsize_, foffset_, &image_)) { LogError("Cannot map the file descriptor"); - amd::Os::CloseFileHandle(fdesc_); + PlatformState::instance().CloseUniqueFileHandle(ufd_); return hipErrorInvalidValue; } image_mapped_ = true; @@ -396,4 +422,4 @@ hipError_t FatBinaryInfo::BuildProgram(const int device_id) { return hipSuccess; } -} //namespace : hip +} //namespace : hip \ No newline at end of file diff --git a/projects/clr/hipamd/src/hip_fatbin.hpp b/projects/clr/hipamd/src/hip_fatbin.hpp index 0d1ca7562b..f9057a4b5a 100644 --- a/projects/clr/hipamd/src/hip_fatbin.hpp +++ b/projects/clr/hipamd/src/hip_fatbin.hpp @@ -28,6 +28,9 @@ THE SOFTWARE. #include "hip_internal.hpp" #include "platform/program.hpp" +// Forward declaration for Unique FD +struct UniqueFD; + namespace hip { //Fat Binary Per Device info @@ -66,7 +69,6 @@ public: hipError_t AddDevProgram(const int device_id); hipError_t BuildProgram(const int device_id); - // Device Id bounds check inline void DeviceIdCheck(const int device_id) const { guarantee(device_id >= 0, "Invalid DeviceId less than 0"); @@ -91,22 +93,24 @@ public: } private: - std::string fname_; // File name - amd::Os::FileDesc fdesc_; // File descriptor - size_t fsize_; // Total file size - size_t foffset_; // File Offset where the fat binary is present. + std::string fname_; //!< File name + amd::Os::FileDesc fdesc_; //!< File descriptor + size_t fsize_; //!< Total file size + size_t foffset_; //!< File Offset where the fat binary is present. // Even when file is passed image will be mmapped till ~desctructor. - const void* image_; // Image - bool image_mapped_; // flag to detect if image is mapped + const void* image_; //!< Image + bool image_mapped_; //!< flag to detect if image is mapped // Only used for FBs where image is directly passed - std::string uri_; // Uniform resource indicator + std::string uri_; //!< Uniform resource indicator // Per Device Info, like corresponding binary ptr, size. std::vector fatbin_dev_info_; + + std::shared_ptr ufd_; //!< Unique file descriptor }; -}; /* namespace hip */ +}; // namespace hip -#endif /* HIP_FAT_BINARY_HPP */ +#endif // HIP_FAT_BINARY_HPP diff --git a/projects/clr/hipamd/src/hip_platform.cpp b/projects/clr/hipamd/src/hip_platform.cpp index 1534d2cdf4..9ed1ee2ea8 100644 --- a/projects/clr/hipamd/src/hip_platform.cpp +++ b/projects/clr/hipamd/src/hip_platform.cpp @@ -897,3 +897,33 @@ void PlatformState::popExec(ihipExec_t& exec) { exec = std::move(hip::tls.exec_stack_.top()); hip::tls.exec_stack_.pop(); } + +std::shared_ptr PlatformState::GetUniqueFileHandle(const std::string& file_path) { + amd::ScopedLock lock(ufd_lock_); + + if (ufd_map_.cend() == ufd_map_.find(file_path)) { + // Get the file desc and file size from amd::Os API + amd::Os::FileDesc fdesc; + size_t fsize = 0; + if (!amd::Os::GetFileHandle(file_path.c_str(), &fdesc, &fsize)) { + return nullptr; + } + ufd_map_.insert(std::make_pair(file_path, std::make_shared(file_path, fdesc, fsize))); + } + + // we should have an entry at this time. + return ufd_map_[file_path]; +} + +bool PlatformState::CloseUniqueFileHandle(const std::shared_ptr& ufd) { + amd::ScopedLock lock(ufd_lock_); + + // if use_count is 2, then there is 1 entry in the map and the current entry is the last close. + if (ufd.use_count() == 2) { + ufd_map_.erase(ufd->fpath_); + if (!amd::Os::CloseFileHandle(ufd->fdesc_)) { + return false; + } + } + return true; +} diff --git a/projects/clr/hipamd/src/hip_platform.hpp b/projects/clr/hipamd/src/hip_platform.hpp index 109a921547..d28db1442b 100644 --- a/projects/clr/hipamd/src/hip_platform.hpp +++ b/projects/clr/hipamd/src/hip_platform.hpp @@ -31,10 +31,23 @@ hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor( hipFunction_t func, int inputBlockSize, size_t dynamicSMemSize, bool bCalcPotentialBlkSz); } /* namespace hip_impl*/ +// Unique file descriptor class +struct UniqueFD { + UniqueFD(const std::string& fpath, amd::Os::FileDesc fdesc, size_t fsize) + : fpath_(fpath), fdesc_(fdesc), fsize_(fsize) {} + + const std::string fpath_; //!< File path of this unique file + const amd::Os::FileDesc fdesc_; //!< File Descriptor + const size_t fsize_; //!< File Size +}; + class PlatformState { amd::Monitor lock_{"Guards PlatformState globals", true}; - /* Singleton object */ + // global level lock for unique file descritor map: ufd_map_ + amd::Monitor ufd_lock_{"Unique FD Store Lock", true}; + + // Singleton object static PlatformState* platform_; PlatformState() {} ~PlatformState() {} @@ -55,7 +68,7 @@ class PlatformState { hipError_t getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr, size_t* size_ptr); - /* Singleton instance */ + // Singleton instance static PlatformState& instance() { if (platform_ == nullptr) { // __hipRegisterFatBinary() will call this when app starts, thus @@ -87,10 +100,17 @@ class PlatformState { void configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream); void popExec(ihipExec_t& exec); + std::shared_ptr GetUniqueFileHandle(const std::string& file_path); + bool CloseUniqueFileHandle(const std::shared_ptr& ufd); + + size_t UfdMapSize() const { return ufd_map_.size(); } + private: // Dynamic Code Object map, keyin module to get the corresponding object std::unordered_map dynCO_map_; - hip::StatCO statCO_; // Static Code object var + hip::StatCO statCO_; //!< Static Code object var bool initialized_{false}; std::unordered_map> texRef_map_; + + std::unordered_map> ufd_map_; //!< Unique File Desc Map };