AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-417244 - Create Unique File Desc Store and guarantee no duplicates fds are opened.

Browse Source 
Change-Id: I7be7fee52b673f0b1b3facdfbd847b57f2a47cde


[ROCm/clr commit: 85b645731c]
This commit is contained in:
kjayapra-amd
2023-08-17 14:59:55 -04:00
committed by Karthik Jayaprakash
parent c0bb71bd74
commit 22bc3d52a3
4 changed files with 117 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/hipamd/src/hip_fatbin.cpp
+50 -24
View File
@@ -24,6 +24,7 @@ THE SOFTWARE.
#include <unordered_map>
#include "hip_code_object.hpp"
#include "hip_platform.hpp"
namespace hip {
@@ -50,28 +51,50 @@ FatBinaryInfo::FatBinaryInfo(const char* fname, const void* image) : fdesc_(amd:
FatBinaryInfo::~FatBinaryInfo() {
for (auto* fbd: fatbin_dev_info_) {
if (fbd != nullptr) {
delete fbd;
if (!HIP_USE_RUNTIME_UNBUNDLER) {
// Using COMGR Unbundler
if (ufd_ && ufd_->fdesc_ > 0) {
// Check for ufd_ != nullptr, since sometimes, we never create unique_file_desc.
if (ufd_->fsize_ && image_mapped_
&& !amd::Os::MemoryUnmapFile(image_, ufd_->fsize_)) {
LogPrintfError("Cannot unmap file for fdesc: %d fsize: %d", ufd_->fdesc_, ufd_->fsize_);
assert(false);
}
if (!PlatformState::instance().CloseUniqueFileHandle(ufd_)) {
LogPrintfError("Cannot close file for fdesc: %d", ufd_->fdesc_);
assert(false);
}
}
}
if (fdesc_ > 0) {
if (fsize_
&& (HIP_USE_RUNTIME_UNBUNDLER || image_mapped_)
&& !amd::Os::MemoryUnmapFile(image_, fsize_)) {
guarantee(false, "Cannot unmap file for fdesc: %d fsize: %d \n", fdesc_, fsize_);
}
if (!amd::Os::CloseFileHandle(fdesc_)) {
guarantee(false, "Cannot close file for fdesc: %d \n", fdesc_);
}
}
fname_ = std::string();
fdesc_ = amd::Os::FDescInit();
fsize_ = 0;
image_ = nullptr;
uri_ = std::string();
fname_ = std::string();
fdesc_ = amd::Os::FDescInit();
fsize_ = 0;
image_ = nullptr;
uri_ = std::string();
if (0 == PlatformState::instance().UfdMapSize()) {
LogError("All Unique FDs are closed");
}
} else {
// Using Runtime Unbundler
if (fdesc_ > 0) {
if (fsize_ && !amd::Os::MemoryUnmapFile(image_, fsize_)) {
LogPrintfError("Cannot unmap file for fdesc: %d fsize: %d", fdesc_, fsize_);
assert(false);
}
if (!amd::Os::CloseFileHandle(fdesc_)) {
LogPrintfError("Cannot close file for fdesc: %d", fdesc_);
assert(false);
}
}
fname_ = std::string();
fdesc_ = amd::Os::FDescInit();
fsize_ = 0;
image_ = nullptr;
uri_ = std::string();
}
}
void ListAllDeviceWithNoCOFromBundle(const std::unordered_map<std::string,
@@ -108,18 +131,21 @@ hipError_t FatBinaryInfo::ExtractFatBinaryUsingCOMGR(const std::vector<hip::Devi
// COMGR file slice APIs.
if (fname_.size() > 0) {
// Get File Handle & size of the file.
if (!amd::Os::GetFileHandle(fname_.c_str(), &fdesc_, &fsize_))
ufd_ = PlatformState::instance().GetUniqueFileHandle(fname_.c_str());
if (ufd_ == nullptr) {
return hipErrorFileNotFound;
}
// If the file name exists but the file size is 0, the something wrong with the file or its path
if (fsize_ == 0)
if (ufd_->fsize_ == 0) {
return hipErrorInvalidValue;
}
// If image_ is nullptr, then file path is passed via hipMod* APIs, so map the file.
if (image_ == nullptr) {
if(!amd::Os::MemoryMapFileDesc(fdesc_, fsize_, foffset_, &image_)) {
if(!amd::Os::MemoryMapFileDesc(ufd_->fdesc_, ufd_->fsize_, foffset_, &image_)) {
LogError("Cannot map the file descriptor");
amd::Os::CloseFileHandle(fdesc_);
PlatformState::instance().CloseUniqueFileHandle(ufd_);
return hipErrorInvalidValue;
}
image_mapped_ = true;
@@ -396,4 +422,4 @@ hipError_t FatBinaryInfo::BuildProgram(const int device_id) {
return hipSuccess;
}
} //namespace : hip
} //namespace : hip
projects/clr/hipamd/src/hip_fatbin.hpp
+14 -10
View File
@@ -28,6 +28,9 @@ THE SOFTWARE.
#include "hip_internal.hpp"
#include "platform/program.hpp"
// Forward declaration for Unique FD
struct UniqueFD;
namespace hip {
//Fat Binary Per Device info
@@ -66,7 +69,6 @@ public:
hipError_t AddDevProgram(const int device_id);
hipError_t BuildProgram(const int device_id);
// Device Id bounds check
inline void DeviceIdCheck(const int device_id) const {
guarantee(device_id >= 0, "Invalid DeviceId less than 0");
@@ -91,22 +93,24 @@ public:
}
private:
std::string fname_; // File name
amd::Os::FileDesc fdesc_; // File descriptor
size_t fsize_; // Total file size
size_t foffset_; // File Offset where the fat binary is present.
std::string fname_; //!< File name
amd::Os::FileDesc fdesc_; //!< File descriptor
size_t fsize_; //!< Total file size
size_t foffset_; //!< File Offset where the fat binary is present.
// Even when file is passed image will be mmapped till ~desctructor.
const void* image_; // Image
bool image_mapped_; // flag to detect if image is mapped
const void* image_; //!< Image
bool image_mapped_; //!< flag to detect if image is mapped
// Only used for FBs where image is directly passed
std::string uri_; // Uniform resource indicator
std::string uri_; //!< Uniform resource indicator
// Per Device Info, like corresponding binary ptr, size.
std::vector<FatBinaryDeviceInfo*> fatbin_dev_info_;
std::shared_ptr<UniqueFD> ufd_; //!< Unique file descriptor
};
}; /* namespace hip */
}; // namespace hip
#endif /* HIP_FAT_BINARY_HPP */
#endif // HIP_FAT_BINARY_HPP
projects/clr/hipamd/src/hip_platform.cpp
+30
View File
@@ -897,3 +897,33 @@ void PlatformState::popExec(ihipExec_t& exec) {
exec = std::move(hip::tls.exec_stack_.top());
hip::tls.exec_stack_.pop();
}
std::shared_ptr<UniqueFD> PlatformState::GetUniqueFileHandle(const std::string& file_path) {
amd::ScopedLock lock(ufd_lock_);
if (ufd_map_.cend() == ufd_map_.find(file_path)) {
// Get the file desc and file size from amd::Os API
amd::Os::FileDesc fdesc;
size_t fsize = 0;
if (!amd::Os::GetFileHandle(file_path.c_str(), &fdesc, &fsize)) {
return nullptr;
}
ufd_map_.insert(std::make_pair(file_path, std::make_shared<UniqueFD>(file_path, fdesc, fsize)));
}
// we should have an entry at this time.
return ufd_map_[file_path];
}
bool PlatformState::CloseUniqueFileHandle(const std::shared_ptr<UniqueFD>& ufd) {
amd::ScopedLock lock(ufd_lock_);
// if use_count is 2, then there is 1 entry in the map and the current entry is the last close.
if (ufd.use_count() == 2) {
ufd_map_.erase(ufd->fpath_);
if (!amd::Os::CloseFileHandle(ufd->fdesc_)) {
return false;
}
}
return true;
}
projects/clr/hipamd/src/hip_platform.hpp
+23 -3
View File
@@ -31,10 +31,23 @@ hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(
hipFunction_t func, int inputBlockSize, size_t dynamicSMemSize, bool bCalcPotentialBlkSz);
} /* namespace hip_impl*/
// Unique file descriptor class
struct UniqueFD {
UniqueFD(const std::string& fpath, amd::Os::FileDesc fdesc, size_t fsize)
: fpath_(fpath), fdesc_(fdesc), fsize_(fsize) {}
const std::string fpath_; //!< File path of this unique file
const amd::Os::FileDesc fdesc_; //!< File Descriptor
const size_t fsize_; //!< File Size
};
class PlatformState {
amd::Monitor lock_{"Guards PlatformState globals", true};
/* Singleton object */
// global level lock for unique file descritor map: ufd_map_
amd::Monitor ufd_lock_{"Unique FD Store Lock", true};
// Singleton object
static PlatformState* platform_;
PlatformState() {}
~PlatformState() {}
@@ -55,7 +68,7 @@ class PlatformState {
hipError_t getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr,
size_t* size_ptr);
/* Singleton instance */
// Singleton instance
static PlatformState& instance() {
if (platform_ == nullptr) {
// __hipRegisterFatBinary() will call this when app starts, thus
@@ -87,10 +100,17 @@ class PlatformState {
void configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream);
void popExec(ihipExec_t& exec);
std::shared_ptr<UniqueFD> GetUniqueFileHandle(const std::string& file_path);
bool CloseUniqueFileHandle(const std::shared_ptr<UniqueFD>& ufd);
size_t UfdMapSize() const { return ufd_map_.size(); }
private:
// Dynamic Code Object map, keyin module to get the corresponding object
std::unordered_map<hipModule_t, hip::DynCO*> dynCO_map_;
hip::StatCO statCO_; // Static Code object var
hip::StatCO statCO_; //!< Static Code object var
bool initialized_{false};
std::unordered_map<textureReference*, std::pair<hipModule_t, std::string>> texRef_map_;
std::unordered_map<std::string, std::shared_ptr<UniqueFD>> ufd_map_; //!< Unique File Desc Map
};