Merge commit 'c60888b0cd135739477e62d400a61138333f1b08' into develop
此提交包含在：
systems-assistant[bot]
+452
-322
@@ -20,7 +20,11 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "hip/hip_runtime_api.h"
|
||||
#include "llvm/BinaryFormat/ELF.h"
|
||||
|
||||
#include "hip_fatbin.hpp"
|
||||
#include "hip_global.hpp"
|
||||
|
||||
#include <unordered_map>
|
||||
#include "hip_code_object.hpp"
|
||||
@@ -28,7 +32,6 @@ THE SOFTWARE.
|
||||
#include "comgrctx.hpp"
|
||||
|
||||
namespace hip {
|
||||
|
||||
namespace comgr_helper {
|
||||
|
||||
template <typename comgr_T> class ComgrUniqueHandle {
|
||||
@@ -154,30 +157,312 @@ void ListAllDeviceWithNoCOFromBundle(
|
||||
}
|
||||
}
|
||||
|
||||
hipError_t FatBinaryInfo::ExtractFatBinaryUsingCOMGR(const std::vector<hip::Device*>& devices,
|
||||
bool &containGenericTarget) {
|
||||
amd_comgr_data_t data_object {0};
|
||||
amd_comgr_status_t comgr_status = AMD_COMGR_STATUS_SUCCESS;
|
||||
hipError_t hip_status = hipSuccess;
|
||||
static std::string TargetGenericMap(const std::string& input) {
|
||||
const static std::unordered_map<std::string, std::string> target_map{
|
||||
// clang-format off
|
||||
{"amdgcn-amd-amdhsa--gfx900" , "amdgcn-amd-amdhsa--gfx9-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx902" , "amdgcn-amd-amdhsa--gfx9-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx904" , "amdgcn-amd-amdhsa--gfx9-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx906" , "amdgcn-amd-amdhsa--gfx9-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx909" , "amdgcn-amd-amdhsa--gfx9-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx90c" , "amdgcn-amd-amdhsa--gfx9-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx942" , "amdgcn-amd-amdhsa--gfx9-4-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx950" , "amdgcn-amd-amdhsa--gfx9-4-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1010", "amdgcn-amd-amdhsa--gfx10-1-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1011", "amdgcn-amd-amdhsa--gfx10-1-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1012", "amdgcn-amd-amdhsa--gfx10-1-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1013", "amdgcn-amd-amdhsa--gfx10-1-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1030", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1031", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1032", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1033", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1034", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1035", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1036", "amdgcn-amd-amdhsa--gfx10-3-generic"},
|
||||
{"amdgcn-amd-amdhsa--gfx1100", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1101", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1102", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1103", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1150", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1151", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1152", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1153", "amdgcn-amd-amdhsa--gfx11-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1200", "amdgcn-amd-amdhsa--gfx12-generic" },
|
||||
{"amdgcn-amd-amdhsa--gfx1201", "amdgcn-amd-amdhsa--gfx12-generic" },
|
||||
// clang-format on
|
||||
};
|
||||
if (auto i = target_map.find(input); i != target_map.end()) {
|
||||
return i->second;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// If image was passed as a pointer to our hipMod* api, we can try to extract the file name
|
||||
// if it was mapped by the app. Otherwise use the COMGR data API.
|
||||
if (fname_.size() == 0) {
|
||||
if (image_ == nullptr) {
|
||||
LogError("Both Filename and image cannot be null");
|
||||
return hipErrorInvalidValue;
|
||||
// For sramecc and xnack
|
||||
static std::string TargetFeatureCheck(const std::string& input, std::string feature) {
|
||||
if (input.find(feature) != std::string::npos) {
|
||||
auto feature_p = feature + "+"; // feature present eg: xnack+
|
||||
auto feature_m = feature + "-"; // feature absent eg: xnack-
|
||||
if (input.find(feature_p) != std::string::npos) {
|
||||
return feature_p;
|
||||
} else if (input.find(feature_m) != std::string::npos) {
|
||||
return feature_m;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
static std::string TargetToGeneric(std::string input) {
|
||||
auto sramecc = TargetFeatureCheck(input, "sramecc");
|
||||
auto xnack = TargetFeatureCheck(input, "xnack");
|
||||
|
||||
// Remove all features
|
||||
size_t index = input.find_first_of(":");
|
||||
std::string name_without_feature = input.substr(0, index);
|
||||
|
||||
// Look up generic name
|
||||
auto generic_name = TargetGenericMap(name_without_feature);
|
||||
if (generic_name.empty()) {
|
||||
return generic_name; // No generic exists
|
||||
}
|
||||
|
||||
// reappend feature
|
||||
if (!sramecc.empty()) {
|
||||
generic_name += ":";
|
||||
generic_name += sramecc;
|
||||
}
|
||||
if (!xnack.empty()) {
|
||||
generic_name += ":";
|
||||
generic_name += xnack;
|
||||
}
|
||||
return generic_name;
|
||||
}
|
||||
|
||||
static bool IsCodeObjectUncompressed(const void* image) {
|
||||
return std::memcmp(image,
|
||||
reinterpret_cast<const void*>(symbols::kOffloadBundleUncompressedMagicStr),
|
||||
sizeof(symbols::kOffloadBundleUncompressedMagicStr) - 1) == 0;
|
||||
}
|
||||
|
||||
static bool IsCodeObjectCompressed(const void* image) {
|
||||
return std::memcmp(image,
|
||||
reinterpret_cast<const void*>(symbols::kOffloadBundleCompressedMagicStr),
|
||||
sizeof(symbols::kOffloadBundleCompressedMagicStr) - 1) == 0;
|
||||
}
|
||||
|
||||
static bool IsCodeObjectElf(const void* image) {
|
||||
const amd::Elf64_Ehdr* ehdr = reinterpret_cast<const amd::Elf64_Ehdr*>(image);
|
||||
return ehdr->e_machine == EM_AMDGPU && ehdr->e_ident[EI_OSABI] == llvm::ELF::ELFOSABI_AMDGPU_HSA;
|
||||
}
|
||||
|
||||
static bool UncompressAndPopulateCodeObject(
|
||||
const void* image, const std::set<std::string>& unique_isa_names,
|
||||
std::map<std::string, std::pair<const void*, size_t>>& code_obj_map) {
|
||||
auto remove_file_extension = [](const std::string& input) -> std::string {
|
||||
size_t index = input.find_last_of(".");
|
||||
std::string ret = input.substr(0, index);
|
||||
return ret;
|
||||
};
|
||||
|
||||
std::vector<std::string> bundle_ids_str;
|
||||
std::set<std::string> unique_ids;
|
||||
|
||||
for (const auto& isa_name : unique_isa_names) {
|
||||
bundle_ids_str.push_back(std::string(symbols::kOffloadKindHipv4_) + isa_name);
|
||||
}
|
||||
|
||||
std::vector<const char*> bundle_ids;
|
||||
bundle_ids.reserve(bundle_ids_str.size());
|
||||
for (auto& bundle_id_str : bundle_ids_str) {
|
||||
bundle_ids.push_back(bundle_id_str.c_str());
|
||||
}
|
||||
|
||||
const auto obheader =
|
||||
reinterpret_cast<const symbols::ClangOffloadBundleCompressedHeader*>(image);
|
||||
const size_t size = obheader->totalSize;
|
||||
|
||||
bool passed = false;
|
||||
do {
|
||||
comgr_helper::ComgrDataSetUniqueHandle bundled_co, unbundled_co;
|
||||
comgr_helper::ComgrDataUniqueHandle input_bundle;
|
||||
if (auto comgr_status = bundled_co.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in creating bundled_co");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = unbundled_co.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in creating unbundled_co");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = input_bundle.Create(AMD_COMGR_DATA_KIND_OBJ_BUNDLE);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in creating input bundle");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::set_data(input_bundle.get(), size, static_cast<const char*>(image));
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in setting image data to bundle");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = amd::Comgr::set_data_name(input_bundle.get(), symbols::kHipFatBinName);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in setting image data to bundle");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = amd::Comgr::data_set_add(bundled_co.get(), input_bundle.get());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in adding data set");
|
||||
break;
|
||||
}
|
||||
|
||||
comgr_helper::ComgrActionInfoUniqueHandle unbundle_action;
|
||||
if (auto comgr_status = unbundle_action.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in creating unbundle action");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = amd::Comgr::action_info_set_bundle_entry_ids(
|
||||
unbundle_action.get(), bundle_ids.data(), bundle_ids.size());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Error in setting bundle entry ids");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = amd::Comgr::do_action(AMD_COMGR_ACTION_UNBUNDLE, unbundle_action.get(),
|
||||
bundled_co.get(), unbundled_co.get());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to unbundle code object");
|
||||
break;
|
||||
}
|
||||
|
||||
size_t count = 0;
|
||||
if (auto comgr_status = amd::Comgr::action_data_count(unbundled_co.get(),
|
||||
AMD_COMGR_DATA_KIND_EXECUTABLE, &count);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get data count of unbundled code object");
|
||||
break;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
amd_comgr_data_t item;
|
||||
if (auto comgr_status = amd::Comgr::action_data_get_data(
|
||||
unbundled_co.get(), AMD_COMGR_DATA_KIND_EXECUTABLE, i, &item);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get data unbundled code object");
|
||||
break;
|
||||
}
|
||||
|
||||
size_t item_name_size = 0;
|
||||
if (auto comgr_status = amd::Comgr::get_data_name(item, &item_name_size, nullptr);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get data size");
|
||||
break;
|
||||
}
|
||||
|
||||
std::string item_bundle_id(item_name_size, 0);
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::get_data_name(item, &item_name_size, item_bundle_id.data());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get data");
|
||||
break;
|
||||
}
|
||||
|
||||
size_t item_size = 0;
|
||||
if (auto comgr_status = amd::Comgr::get_data(item, &item_size, nullptr);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get data size");
|
||||
break;
|
||||
}
|
||||
|
||||
if (item_size > 0) {
|
||||
char* item_data = new char[item_size];
|
||||
if (auto comgr_status = amd::Comgr::get_data(item, &item_size, item_data);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get data");
|
||||
break;
|
||||
}
|
||||
|
||||
std::string bundle_entry = remove_file_extension(
|
||||
std::string(item_bundle_id.c_str() + sizeof(symbols::kOffloadHipV4FatBinName_) - 1));
|
||||
LogPrintfInfo("Inserting bundle entry of %s : size: %d, data: %p", bundle_entry.c_str(),
|
||||
item_size, item_data);
|
||||
code_obj_map[bundle_entry] = std::make_pair(item_data, item_size);
|
||||
}
|
||||
}
|
||||
passed = true;
|
||||
} while (0);
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
static bool PopulateCodeObjectMap(
|
||||
const void* image, const std::set<std::string>& unique_isa_names,
|
||||
std::map<std::string, std::pair<const void*, size_t>>& code_obj_map) {
|
||||
bool passed = false;
|
||||
do {
|
||||
comgr_helper::ComgrDataUniqueHandle data_object;
|
||||
if (auto comgr_status = data_object.Create(AMD_COMGR_DATA_KIND_FATBIN);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Creating data object failed with status %d ", comgr_status);
|
||||
break;
|
||||
}
|
||||
|
||||
// There is no way to find size of offload bundle, so we pass 4096 here.
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::set_data(data_object.get(), 4096, reinterpret_cast<const char*>(image));
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Setting data from file slice failed with status %d ", comgr_status);
|
||||
break;
|
||||
}
|
||||
|
||||
// Create a query list using COMGR info for unique ISAs.
|
||||
std::vector<amd_comgr_code_object_info_t> query_list_array;
|
||||
query_list_array.reserve(unique_isa_names.size());
|
||||
for (const auto& isa_name : unique_isa_names) {
|
||||
auto& item = query_list_array.emplace_back();
|
||||
item.isa = isa_name.c_str();
|
||||
item.size = 0;
|
||||
item.offset = 0;
|
||||
}
|
||||
|
||||
// Look up the code object info passing the query list.
|
||||
if (auto comgr_status = amd::Comgr::lookup_code_object(
|
||||
data_object.get(), query_list_array.data(), unique_isa_names.size());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Setting data from file slice failed with status %d ", comgr_status);
|
||||
break;
|
||||
}
|
||||
|
||||
for (const auto& item : query_list_array) {
|
||||
if (item.size > 0) {
|
||||
char* d = new char[item.size];
|
||||
std::memcpy(reinterpret_cast<void*>(d), reinterpret_cast<const char*>(image) + item.offset,
|
||||
item.size);
|
||||
code_obj_map[item.isa] = std::make_pair(d, item.size);
|
||||
}
|
||||
}
|
||||
|
||||
passed = true;
|
||||
} while (0);
|
||||
return passed;
|
||||
}
|
||||
|
||||
hipError_t FatBinaryInfo::ExtractFatBinaryUsingCOMGR(const std::vector<hip::Device*>& devices) {
|
||||
if (fname_.empty() && image_ == nullptr) {
|
||||
LogError("Both Filename and image cannot be null");
|
||||
return hipErrorInvalidValue;
|
||||
}
|
||||
|
||||
if (image_ != nullptr) {
|
||||
if (!amd::Os::FindFileNameFromAddress(image_, &fname_, &foffset_)) {
|
||||
fname_ = std::string("");
|
||||
foffset_ = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// If file name & path are available (or it is passed to you), then get the file desc to use
|
||||
// COMGR file slice APIs.
|
||||
if (image_ == nullptr && fname_.size() > 0) {
|
||||
// Get File Handle & size of the file.
|
||||
} else {
|
||||
ufd_ = PlatformState::instance().GetUniqueFileHandle(fname_.c_str());
|
||||
if (ufd_ == nullptr) {
|
||||
return hipErrorFileNotFound;
|
||||
@@ -197,264 +482,183 @@ hipError_t FatBinaryInfo::ExtractFatBinaryUsingCOMGR(const std::vector<hip::Devi
|
||||
|
||||
image_mapped_ = true;
|
||||
}
|
||||
|
||||
// At this line, image should be a valid ptr.
|
||||
guarantee(image_ != nullptr, "Image cannot be nullptr, file:%s did not map for some reason",
|
||||
fname_.c_str());
|
||||
|
||||
bool is_compressed = IsCodeObjectCompressed(image_),
|
||||
is_uncompressed = IsCodeObjectUncompressed(image_);
|
||||
|
||||
// It better be elf if its neither compressed nor uncompressed
|
||||
if (!is_compressed && !is_uncompressed) {
|
||||
if (IsCodeObjectElf(image_)) {
|
||||
// Load the binary directly
|
||||
auto elf_size = amd::Elf::getElfSize(image_);
|
||||
for (size_t i = 0; i < devices.size(); i++) {
|
||||
if (hipSuccess != AddDevProgram(devices[i], image_, elf_size, 0))
|
||||
return hipErrorInvalidImage;
|
||||
}
|
||||
return hipSuccess; // We are done since it was already ELF
|
||||
} else {
|
||||
LogError("The code object has invalid header: compressed, uncompressed or elf");
|
||||
return hipErrorInvalidImage;
|
||||
}
|
||||
}
|
||||
|
||||
// Create a list of all targets, which the current device can run
|
||||
// For example, gfx1030 can run gfx1030, gfx10-geneeric, amdgcnspirv
|
||||
std::set<std::string> unique_isa_names;
|
||||
const std::string spirv_isa_name{"spirv64-amd-amdhsa--amdgcnspirv"};
|
||||
unique_isa_names.insert(spirv_isa_name); // Insert SPIRV ISA name
|
||||
for (auto device : devices) {
|
||||
std::string device_name = device->devices()[0]->isa().isaName();
|
||||
unique_isa_names.insert(device_name);
|
||||
auto generic_name = TargetToGeneric(device_name);
|
||||
LogPrintfInfo("Looking up generic name of : %s - %s", device_name.c_str(),
|
||||
generic_name.c_str());
|
||||
if (!generic_name.empty()) {
|
||||
unique_isa_names.insert(generic_name);
|
||||
}
|
||||
}
|
||||
|
||||
std::map<std::string, std::pair<const void*, size_t>> code_obj_map; //!< code object map
|
||||
if (is_compressed) {
|
||||
if (!UncompressAndPopulateCodeObject(image_, unique_isa_names, code_obj_map)) {
|
||||
return hipErrorInvalidImage;
|
||||
}
|
||||
} else { // uncompressed code object
|
||||
if (!PopulateCodeObjectMap(image_, unique_isa_names, code_obj_map)) {
|
||||
return hipErrorInvalidImage;
|
||||
}
|
||||
}
|
||||
|
||||
hipError_t hip_status = hipErrorInvalidImage;
|
||||
do {
|
||||
bool isCompressed = false;
|
||||
// If the image ptr is not clang offload bundle then just directly point the image.
|
||||
if (!CodeObject::IsClangOffloadMagicBundle(image_, isCompressed)) {
|
||||
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
|
||||
uint64_t elf_size = CodeObject::ElfSize(image_);
|
||||
if (elf_size == 0) {
|
||||
hip_status = hipErrorInvalidImage;
|
||||
break;
|
||||
}
|
||||
hip_status = AddDevProgram(devices[dev_idx], image_, elf_size, 0);
|
||||
bool spirv_isa_found = code_obj_map.find(spirv_isa_name) != code_obj_map.end();
|
||||
for (auto device : devices) {
|
||||
std::string device_name = device->devices()[0]->isa().isaName();
|
||||
auto generic_target_name = TargetToGeneric(device_name); // Generic Code Object
|
||||
auto native_co = code_obj_map.find(device_name); // Native Code Object
|
||||
auto generic_co = code_obj_map.find(generic_target_name); // generic Code Object
|
||||
LogPrintfInfo("Device name: %s Generic name: %s", device_name.c_str(),
|
||||
generic_target_name.c_str());
|
||||
|
||||
// If the size is not 0, that means we found the native isa code object
|
||||
if (native_co != code_obj_map.end() && !HIP_FORCE_SPIRV_CODEOBJECT) {
|
||||
LogPrintfInfo("Using Native code object: %s", device->devices()[0]->isa().targetId());
|
||||
|
||||
// We need to do this because there is existing mechanism which deletes code object in
|
||||
// destructor. Ideally next set of refactor should sort it.
|
||||
char* co = new char[native_co->second.second];
|
||||
std::memcpy(co, reinterpret_cast<const char*>(native_co->second.first),
|
||||
native_co->second.second);
|
||||
hip_status = AddDevProgram(device, co, native_co->second.second, 0);
|
||||
if (hip_status != hipSuccess) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!isCompressed) {
|
||||
if (CodeObject::containGenericTarget(image_)) {
|
||||
LogInfo("offload bundle contains generic target code object");
|
||||
containGenericTarget = true;
|
||||
}
|
||||
}
|
||||
if (isCompressed || containGenericTarget) {
|
||||
size_t major = 0, minor = 0;
|
||||
amd::Comgr::get_version(&major, &minor);
|
||||
if ((major == 2 && minor >= 8) || major > 2) {
|
||||
hip_status = ExtractFatBinaryUsingCOMGR(image_, devices);
|
||||
break;
|
||||
} else if (isCompressed) {
|
||||
LogPrintfError("comgr %zu.%zu cannot support compressed mode which requires comgr 2.8+",
|
||||
major, minor);
|
||||
hip_status = hipErrorNotSupported;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Create a data object, if it fails return error
|
||||
if ((comgr_status = amd::Comgr::create_data(AMD_COMGR_DATA_KIND_FATBIN, &data_object)) !=
|
||||
AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Creating data object failed with status %d ", comgr_status);
|
||||
hip_status = hipErrorInvalidValue;
|
||||
break;
|
||||
}
|
||||
|
||||
if (ufd_ != nullptr && amd::Os::isValidFileDesc(ufd_->fdesc_)) {
|
||||
LogPrintfError("Have valid file!%d", ufd_->fdesc_);
|
||||
}
|
||||
|
||||
if (image_ != nullptr) {
|
||||
// Using the image ptr, map the data object.
|
||||
if ((comgr_status =
|
||||
amd::Comgr::set_data(data_object, 4096, reinterpret_cast<const char*>(image_))) !=
|
||||
AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Setting data from file slice failed with status %d ", comgr_status);
|
||||
hip_status = hipErrorInvalidValue;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
guarantee(false, "Cannot have both fname_ and image_ as nullptr");
|
||||
}
|
||||
|
||||
// Find the unique number of ISAs needed for this COMGR query.
|
||||
std::unordered_map<std::string, std::pair<size_t, size_t>> unique_isa_names;
|
||||
for (auto device : devices) {
|
||||
std::string device_name = device->devices()[0]->isa().isaName();
|
||||
unique_isa_names.insert({device_name, std::make_pair<size_t, size_t>(0, 0)});
|
||||
}
|
||||
|
||||
// there are two spirv targets, spirv64-amd-amdhsa--amdgcnspirv and
|
||||
// spirv64-amd-amdhsa-unknown-amdgcnspirv.
|
||||
// eventually we will remove spirv64-amd-amdhsa--amdgcnspirv
|
||||
const std::vector<std::string> spirv_isa_names = {"spirv64-amd-amdhsa--amdgcnspirv",
|
||||
"spirv64-amd-amdhsa-unknown-amdgcnspirv"};
|
||||
for (const auto& spirv_isa_name : spirv_isa_names) {
|
||||
unique_isa_names.insert({spirv_isa_name, std::make_pair<size_t, size_t>(0, 0)});
|
||||
}
|
||||
|
||||
// Create a query list using COMGR info for unique ISAs.
|
||||
std::vector<amd_comgr_code_object_info_t> query_list_array;
|
||||
query_list_array.reserve(unique_isa_names.size());
|
||||
for (const auto& isa_name : unique_isa_names) {
|
||||
auto& item = query_list_array.emplace_back();
|
||||
item.isa = isa_name.first.c_str();
|
||||
item.size = 0;
|
||||
item.offset = 0;
|
||||
}
|
||||
|
||||
// Look up the code object info passing the query list.
|
||||
if ((comgr_status = amd::Comgr::lookup_code_object(data_object, query_list_array.data(),
|
||||
unique_isa_names.size())) !=
|
||||
AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Setting data from file slice failed with status %d ", comgr_status);
|
||||
hip_status = hipErrorInvalidValue;
|
||||
break;
|
||||
}
|
||||
|
||||
for (const auto& item : query_list_array) {
|
||||
auto unique_it = unique_isa_names.find(item.isa);
|
||||
guarantee(unique_isa_names.cend() != unique_it, "Cannot find unique isa ");
|
||||
unique_it->second = std::pair<size_t, size_t>(static_cast<size_t>(item.size),
|
||||
static_cast<size_t>(item.offset));
|
||||
}
|
||||
|
||||
bool spirv_isa_found = false;
|
||||
decltype(unique_isa_names.begin()) spirv_isa_handle;
|
||||
for (const auto& spirv_isa_name : spirv_isa_names) {
|
||||
auto iter = unique_isa_names.find(spirv_isa_name);
|
||||
if (iter->second.first != 0) {
|
||||
spirv_isa_found = true;
|
||||
spirv_isa_handle = iter;
|
||||
}
|
||||
}
|
||||
bool get_spirv_data_res = false;
|
||||
std::once_flag get_spirv_data_flag;
|
||||
std::unordered_map<std::string, std::pair<char*, size_t>> compiled_co; // code object cache
|
||||
comgr_helper::ComgrDataSetUniqueHandle spirv_data_set;
|
||||
comgr_helper::ComgrDataUniqueHandle spirv_data;
|
||||
|
||||
auto get_spirv_data = [&]() {
|
||||
if (comgr_status = spirv_data_set.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create SPIRV Data set");
|
||||
return;
|
||||
}
|
||||
|
||||
if (comgr_status = spirv_data.Create(AMD_COMGR_DATA_KIND_SPIRV);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create SPIRV Data");
|
||||
return;
|
||||
}
|
||||
|
||||
if (comgr_status =
|
||||
amd::Comgr::set_data(spirv_data.get(), spirv_isa_handle->second.first /* size */,
|
||||
reinterpret_cast<char*>(const_cast<void*>(image_)) +
|
||||
spirv_isa_handle->second.second /* buffer */);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to assign data in comgr");
|
||||
return;
|
||||
}
|
||||
|
||||
if (comgr_status = amd::Comgr::set_data_name(spirv_data.get(), "hip_code_object.spv");
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set data name");
|
||||
return;
|
||||
}
|
||||
|
||||
if (comgr_status = amd::Comgr::data_set_add(spirv_data_set.get(), spirv_data.get());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to add spir data");
|
||||
return;
|
||||
}
|
||||
|
||||
get_spirv_data_res = true;
|
||||
};
|
||||
|
||||
LogPrintfInfo("Searching for code objects, HIP_FORCE_SPIRV_CODEOBJECT: %d",
|
||||
HIP_FORCE_SPIRV_CODEOBJECT);
|
||||
|
||||
for (auto device : devices) {
|
||||
std::string device_name = device->devices()[0]->isa().isaName();
|
||||
auto dev_it = unique_isa_names.find(device_name);
|
||||
// If the size is not 0, that means we found the native isa code object
|
||||
if (dev_it->second.first != 0 && !HIP_FORCE_SPIRV_CODEOBJECT) {
|
||||
LogPrintfInfo("Using Native code object: %s", device->devices()[0]->isa().targetId());
|
||||
guarantee(unique_isa_names.cend() != dev_it,
|
||||
"Cannot find the device name in the unique device name");
|
||||
hip_status = AddDevProgram(
|
||||
device, reinterpret_cast<address>(const_cast<void*>(image_)) + dev_it->second.second,
|
||||
dev_it->second.first, dev_it->second.second);
|
||||
} else if (generic_co != code_obj_map.end() && !HIP_FORCE_SPIRV_CODEOBJECT) {
|
||||
LogPrintfInfo("Using Generic code object: %s : %s", device->devices()[0]->isa().targetId(),
|
||||
generic_target_name.c_str());
|
||||
char* co = new char[generic_co->second.second];
|
||||
std::memcpy(co, reinterpret_cast<const char*>(generic_co->second.first),
|
||||
generic_co->second.second);
|
||||
hip_status = AddDevProgram(device, co, generic_co->second.second, 0);
|
||||
if (hip_status != hipSuccess) {
|
||||
break;
|
||||
}
|
||||
} else if (spirv_isa_found) {
|
||||
std::call_once(get_spirv_data_flag, get_spirv_data);
|
||||
|
||||
if(!get_spirv_data_res) {
|
||||
hip_status = hipErrorInvalidValue;
|
||||
break;
|
||||
}
|
||||
std::string target_id = device->devices()[0]->isa().targetId();
|
||||
if (auto code_iter = compiled_co.find(target_id); code_iter != compiled_co.end()) {
|
||||
// We have already compiled for it, lets reuse the code object
|
||||
char* co = new char[code_iter->second.second];
|
||||
std::memcpy(co, code_iter->second.first, code_iter->second.second);
|
||||
LogPrintfInfo("reusing code object for: %s", target_id.c_str());
|
||||
hip_status = AddDevProgram(device, co, code_iter->second.second, 0);
|
||||
if (hip_status != hipSuccess) {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
LogPrintfInfo("Creating ISA for: %s from spirv", target_id.c_str());
|
||||
comgr_helper::ComgrActionInfoUniqueHandle reloc_action;
|
||||
std::string isa = "amdgcn-amd-amdhsa--" + target_id;
|
||||
if (comgr_status = reloc_action.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create action");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status = amd::Comgr::action_info_set_isa_name(reloc_action.get(), isa.c_str());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set ISA name");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status = amd::Comgr::action_info_set_device_lib_linking(reloc_action.get(), true);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set device lib linking");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status = amd::Comgr::action_info_set_option_list(
|
||||
reloc_action.get(), nullptr /* options list */, 0 /* options size */);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set option list");
|
||||
break;
|
||||
}
|
||||
LogPrintfInfo("Creating ISA for: %s from spirv", target_id.c_str());
|
||||
|
||||
comgr_helper::ComgrDataSetUniqueHandle spirv_data_set;
|
||||
comgr_helper::ComgrDataSetUniqueHandle reloc_data;
|
||||
if (comgr_status = reloc_data.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create reloc data set");
|
||||
comgr_helper::ComgrDataUniqueHandle spirv_data;
|
||||
comgr_helper::ComgrActionInfoUniqueHandle reloc_action;
|
||||
|
||||
if (auto comgr_status = spirv_data_set.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create SPIRV Data set");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status =
|
||||
if (auto comgr_status = spirv_data.Create(AMD_COMGR_DATA_KIND_SPIRV);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create SPIRV Data");
|
||||
break;
|
||||
}
|
||||
|
||||
auto spirv_isa_handle = code_obj_map.find(spirv_isa_name);
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::set_data(spirv_data.get(), spirv_isa_handle->second.second /* size */,
|
||||
reinterpret_cast<const char*>(spirv_isa_handle->second.first)
|
||||
/* buffer */);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to assign SPIRV data");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = amd::Comgr::set_data_name(spirv_data.get(), "hip_code_object.spv");
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set spirv data's name");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = amd::Comgr::data_set_add(spirv_data_set.get(), spirv_data.get());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to add spir data to data set");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = reloc_action.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create reloc action");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::action_info_set_isa_name(reloc_action.get(), isa.c_str());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set reloc action's isa name");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status = reloc_data.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create reloc data");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::action_info_set_device_lib_linking(reloc_action.get(), true);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set device lib linking for reloc action");
|
||||
break;
|
||||
}
|
||||
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::do_action(AMD_COMGR_ACTION_COMPILE_SPIRV_TO_RELOCATABLE,
|
||||
reloc_action.get(), spirv_data_set.get(), reloc_data.get());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to compile to reloc");
|
||||
LogError("Failed to do action: codegen bc ot reloc");
|
||||
LogError("Failed to compile spirv to reloc");
|
||||
break;
|
||||
}
|
||||
|
||||
comgr_helper::ComgrActionInfoUniqueHandle exe_action;
|
||||
comgr_helper::ComgrDataSetUniqueHandle exe_output;
|
||||
|
||||
if (comgr_status = exe_action.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create action");
|
||||
if (auto comgr_status = exe_action.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create exe action");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status = amd::Comgr::action_info_set_isa_name(exe_action.get(), isa.c_str());
|
||||
if (auto comgr_status = amd::Comgr::action_info_set_isa_name(exe_action.get(), isa.c_str());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to set exe action isa name");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status = exe_output.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
if (auto comgr_status = exe_output.Create(); comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to create exe output");
|
||||
break;
|
||||
}
|
||||
|
||||
if (comgr_status =
|
||||
if (auto comgr_status =
|
||||
amd::Comgr::do_action(AMD_COMGR_ACTION_LINK_RELOCATABLE_TO_EXECUTABLE,
|
||||
exe_action.get(), reloc_data.get(), exe_output.get());
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
@@ -463,74 +667,51 @@ hipError_t FatBinaryInfo::ExtractFatBinaryUsingCOMGR(const std::vector<hip::Devi
|
||||
}
|
||||
|
||||
amd_comgr_data_t exe_data_handle;
|
||||
if (comgr_status = amd::Comgr::action_data_get_data(
|
||||
if (auto comgr_status = amd::Comgr::action_data_get_data(
|
||||
exe_output.get(), AMD_COMGR_DATA_KIND_EXECUTABLE, 0, &exe_data_handle);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to action get exe data");
|
||||
LogError("Failed to get exe data");
|
||||
break;
|
||||
}
|
||||
|
||||
// Move ownership of exe_data_handle to exe_data
|
||||
comgr_helper::ComgrDataUniqueHandle exe_data(exe_data_handle);
|
||||
|
||||
size_t co_size;
|
||||
if (comgr_status = amd::Comgr::get_data(exe_data.get(), &co_size, NULL);
|
||||
size_t co_size = 0;
|
||||
if (auto comgr_status = amd::Comgr::get_data(exe_data.get(), &co_size, NULL);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get exe size");
|
||||
break;
|
||||
}
|
||||
|
||||
char* co = new char[co_size];
|
||||
if (comgr_status = amd::Comgr::get_data(exe_data.get(), &co_size, co);
|
||||
if (auto comgr_status = amd::Comgr::get_data(exe_data.get(), &co_size, co);
|
||||
comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("Failed to get exe data");
|
||||
break;
|
||||
}
|
||||
|
||||
auto elf_size = CodeObject::ElfSize(co);
|
||||
hip_status = AddDevProgram(device, co, elf_size, 0);
|
||||
hip_status = AddDevProgram(device, co, co_size, 0);
|
||||
if (hip_status != hipSuccess) {
|
||||
break;
|
||||
}
|
||||
// Save the compiled code object
|
||||
compiled_co[target_id] = std::make_pair(co, elf_size);
|
||||
} else {
|
||||
// We found neither a compatible code object nor SPIRV
|
||||
LogPrintfError(
|
||||
"No compatible code objects found for: %s, value of HIP_FORCE_SPIRV_CODEOBJECT: %d",
|
||||
device->devices()[0]->isa().targetId(), HIP_FORCE_SPIRV_CODEOBJECT);
|
||||
hip_status = hipErrorInvalidValue;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} while (0);
|
||||
|
||||
if (comgr_status != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogError("comgr API call failed");
|
||||
hip_status = hipErrorInvalidValue;
|
||||
}
|
||||
|
||||
// Clean up file and memory resouces if hip_status failed for some reason.
|
||||
if (hip_status != hipSuccess && hip_status != hipErrorInvalidKernelFile) {
|
||||
ReleaseImageAndFile();
|
||||
}
|
||||
|
||||
if (data_object.handle) {
|
||||
if ((comgr_status = amd::Comgr::release_data(data_object)) != AMD_COMGR_STATUS_SUCCESS) {
|
||||
LogPrintfError("Releasing COMGR data failed with status %d ", comgr_status);
|
||||
return hipErrorInvalidValue;
|
||||
}
|
||||
// release code objects
|
||||
for (const auto& co : code_obj_map) {
|
||||
delete[] reinterpret_cast<const char*>(co.second.first);
|
||||
}
|
||||
|
||||
return hip_status;
|
||||
}
|
||||
|
||||
hipError_t FatBinaryInfo::ExtractFatBinary(const std::vector<hip::Device*>& devices) {
|
||||
amd::ScopedLock lock(FatBinaryLock());
|
||||
|
||||
bool containGenericTarget = false;
|
||||
return ExtractFatBinaryUsingCOMGR(devices, containGenericTarget);
|
||||
}
|
||||
|
||||
hipError_t FatBinaryInfo::AddDevProgram(hip::Device* device, const void* binary_image,
|
||||
size_t binary_size, size_t binary_offset) {
|
||||
int devID = device->deviceId();
|
||||
@@ -569,55 +750,4 @@ hipError_t FatBinaryInfo::BuildProgram(const int device_id) {
|
||||
}
|
||||
return hipSuccess;
|
||||
}
|
||||
|
||||
// ================================================================================================
|
||||
hipError_t FatBinaryInfo::ExtractFatBinaryUsingCOMGR(const void* data,
|
||||
const std::vector<hip::Device*>& devices) {
|
||||
hipError_t hip_status = hipSuccess;
|
||||
// At this line, image should be a valid ptr.
|
||||
guarantee(data != nullptr, "Image cannot be nullptr");
|
||||
|
||||
do {
|
||||
std::vector<std::pair<const void*, size_t>> code_objs;
|
||||
// Copy device names
|
||||
std::vector<std::string> device_names;
|
||||
device_names.reserve(devices.size());
|
||||
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
|
||||
device_names.push_back(devices[dev_idx]->devices()[0]->isa().isaName());
|
||||
}
|
||||
|
||||
hip_status =
|
||||
CodeObject::extractCodeObjectFromFatBinaryUsingComgr(data, 0, device_names, code_objs);
|
||||
if (hip_status == hipErrorNoBinaryForGpu || hip_status == hipSuccess) {
|
||||
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
|
||||
if (code_objs[dev_idx].first) {
|
||||
hip_status =
|
||||
AddDevProgram(devices[dev_idx], code_objs[dev_idx].first, code_objs[dev_idx].second, 0);
|
||||
if (hip_status != hipSuccess) {
|
||||
return hip_status;
|
||||
}
|
||||
} else {
|
||||
// This is the case of hipErrorNoBinaryForGpu which will finally fail app on device
|
||||
// without code object
|
||||
LogPrintfError("Cannot find CO in the bundle %s for ISA: %s", fname_.c_str(),
|
||||
device_names[dev_idx].c_str());
|
||||
}
|
||||
}
|
||||
} else if (hip_status == hipErrorInvalidKernelFile) {
|
||||
hip_status = hipSuccess;
|
||||
// If the image ptr is not clang offload bundle then just directly point the image.
|
||||
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
|
||||
hip_status = AddDevProgram(devices[dev_idx], data, CodeObject::ElfSize(data), 0);
|
||||
if (hip_status != hipSuccess) {
|
||||
return hip_status;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
LogPrintfError("CodeObject::extractCodeObjectFromFatBinaryUsingComgr failed with status %d\n",
|
||||
hip_status);
|
||||
}
|
||||
} while (0);
|
||||
|
||||
return hip_status;
|
||||
}
|
||||
} // namespace hip
|
||||
|
||||
新增問題並參考
封鎖使用者