Add APIs to support debugging vm fault

1. Add hsa ext api hsa_amd_register_vmfault_handler for debugger to register callback in case of VM fault.
2. Extend hsa_ven_amd_loader API to:
   (1) iterate loaded code objects in executable:
       hsa_ven_amd_loader_executable_iterate_loaded_code_objects
   (2) get loaded code object info:
       hsa_ven_amd_loader_loaded_code_object_get_info
3. Make the id of hsa_queue the same as the one used in communication with thunk (for amd_aql_queue)

Change-Id: I68910809e59e24297350d262606f00e96c14bcbd

runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -171,7 +171,7 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, Scr
   amd_queue_.hsa_queue.base_address = ring_buf_;
   amd_queue_.hsa_queue.doorbell_signal = Signal::Convert(this);
   amd_queue_.hsa_queue.size = queue_size_pkts;
-  amd_queue_.hsa_queue.id = core::Runtime::runtime_singleton_->GetQueueId();
+  amd_queue_.hsa_queue.id = queue_id_;
   amd_queue_.read_dispatch_id_field_base_byte_offset = uint32_t(
       uintptr_t(&amd_queue_.read_dispatch_id) - uintptr_t(&amd_queue_));
 
@@ -893,21 +893,21 @@ void AqlQueue::InitScratchSRD() {
   SQ_BUF_RSRC_WORD1 srd1;
   SQ_BUF_RSRC_WORD2 srd2;
   SQ_BUF_RSRC_WORD3 srd3;
-  
+
   uint32_t scratch_base_hi = 0;
   uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base);
   #ifdef HSA_LARGE_MODEL
   scratch_base_hi = uint32_t(scratch_base >> 32);
   #endif
   srd0.bits.BASE_ADDRESS = uint32_t(scratch_base);
-  
+
   srd1.bits.BASE_ADDRESS_HI = scratch_base_hi;
   srd1.bits.STRIDE = 0;
   srd1.bits.CACHE_SWIZZLE = 0;
   srd1.bits.SWIZZLE_ENABLE = 1;
-  
+
   srd2.bits.NUM_RECORDS = uint32_t(queue_scratch_.size);
-  
+
   srd3.bits.DST_SEL_X = SQ_SEL_X;
   srd3.bits.DST_SEL_Y = SQ_SEL_Y;
   srd3.bits.DST_SEL_Z = SQ_SEL_Z;
@@ -922,8 +922,8 @@ void AqlQueue::InitScratchSRD() {
   srd3.bits.HEAP = 0;
   srd3.bits.MTYPE__CI__VI = 0;
   srd3.bits.TYPE = SQ_RSRC_BUF;
-  
-  // Update Queue's Scratch descriptor's property 
+
+  // Update Queue's Scratch descriptor's property
   amd_queue_.scratch_resource_descriptor[0] = srd0.u32All;
   amd_queue_.scratch_resource_descriptor[1] = srd1.u32All;
   amd_queue_.scratch_resource_descriptor[2] = srd2.u32All;
@@ -949,7 +949,7 @@ void AqlQueue::InitScratchSRD() {
   uint32_t max_scratch_waves = num_cus * agent_props.MaxSlotsScratchCU;
 
   // Scratch is allocated program COMPUTE_TMPRING_SIZE register
-  // Scratch Size per Wave is specified in terms of kilobytes 
+  // Scratch Size per Wave is specified in terms of kilobytes
   uint32_t wave_size = agent_props.WaveFrontSize;
   tmpring_size.bits.WAVESIZE =
         (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024);

runtime/hsa-runtime/core/runtime/host_queue.cpp

@@ -2,24 +2,24 @@
 //
 // The University of Illinois/NCSA
 // Open Source License (NCSA)
-// 
+//
 // Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
-// 
+//
 // Developed by:
-// 
+//
 //                 AMD Research and AMD HSA Software Development
-// 
+//
 //                 Advanced Micro Devices, Inc.
-// 
+//
 //                 www.amd.com
-// 
+//
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to
 // deal with the Software without restriction, including without limitation
 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
 // and/or sell copies of the Software, and to permit persons to whom the
 // Software is furnished to do so, subject to the following conditions:
-// 
+//
 //  - Redistributions of source code must retain the above copyright notice,
 //    this list of conditions and the following disclaimers.
 //  - Redistributions in binary form must reproduce the above copyright
@@ -29,7 +29,7 @@
 //    nor the names of its contributors may be used to endorse or promote
 //    products derived from this Software without specific prior written
 //    permission.
-// 
+//
 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
@@ -46,6 +46,9 @@
 #include "core/util/utils.h"
 
 namespace core {
+
+volatile uint32_t HostQueue::queue_count_ = 0x80000000;
+
 HostQueue::HostQueue(hsa_region_t region, uint32_t ring_size,
                      hsa_queue_type32_t type, uint32_t features,
                      hsa_signal_t doorbell_signal)
@@ -70,7 +73,7 @@ HostQueue::HostQueue(hsa_region_t region, uint32_t ring_size,
   amd_queue_.hsa_queue.base_address = ring_;
   amd_queue_.hsa_queue.size = size_;
   amd_queue_.hsa_queue.doorbell_signal = doorbell_signal;
-  amd_queue_.hsa_queue.id = Runtime::runtime_singleton_->GetQueueId();
+  amd_queue_.hsa_queue.id = atomic::Increment(&queue_count_);
   amd_queue_.hsa_queue.type = type;
   amd_queue_.hsa_queue.features = features;
 #ifdef HSA_LARGE_MODEL

runtime/hsa-runtime/core/runtime/hsa.cpp

@@ -2,24 +2,24 @@
 //
 // The University of Illinois/NCSA
 // Open Source License (NCSA)
-// 
+//
 // Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
-// 
+//
 // Developed by:
-// 
+//
 //                 AMD Research and AMD HSA Software Development
-// 
+//
 //                 Advanced Micro Devices, Inc.
-// 
+//
 //                 www.amd.com
-// 
+//
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to
 // deal with the Software without restriction, including without limitation
 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
 // and/or sell copies of the Software, and to permit persons to whom the
 // Software is furnished to do so, subject to the following conditions:
-// 
+//
 //  - Redistributions of source code must retain the above copyright notice,
 //    this list of conditions and the following disclaimers.
 //  - Redistributions in binary form must reproduce the above copyright
@@ -29,7 +29,7 @@
 //    nor the names of its contributors may be used to endorse or promote
 //    products derived from this Software without specific prior written
 //    permission.
-// 
+//
 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
@@ -219,7 +219,7 @@ hsa_status_t hsa_shut_down() {
 //---------------------------------------------------------------------------//
 //  System
 //---------------------------------------------------------------------------//
-hsa_status_t 
+hsa_status_t
     hsa_system_get_info(hsa_system_info_t attribute, void* value) {
   TRY;
   IS_OPEN();
@@ -262,7 +262,7 @@ hsa_status_t hsa_extension_get_name(uint16_t extension, const char** name) {
   CATCH;
 }
 
-hsa_status_t 
+hsa_status_t
     hsa_system_extension_supported(uint16_t extension, uint16_t version_major,
                                    uint16_t version_minor, bool* result) {
   TRY;
@@ -342,6 +342,7 @@ static size_t get_extension_table_length(uint16_t extension, uint16_t major, uin
       {"hsa_ext_images_1_00_pfn_t", sizeof(hsa_ext_images_1_00_pfn_t)},
       {"hsa_ext_finalizer_1_00_pfn_t", sizeof(hsa_ext_finalizer_1_00_pfn_t)},
       {"hsa_ven_amd_loader_1_00_pfn_t", sizeof(hsa_ven_amd_loader_1_00_pfn_t)},
+      {"hsa_ven_amd_loader_1_01_pfn_t", sizeof(hsa_ven_amd_loader_1_01_pfn_t)},
       {"hsa_ven_amd_aqlprofile_1_00_pfn_t", sizeof(hsa_ven_amd_aqlprofile_1_00_pfn_t)}};
   static const size_t num_tables = sizeof(sizes) / sizeof(sizes_t);
 
@@ -448,12 +449,16 @@ hsa_status_t hsa_system_get_major_extension_table(uint16_t extension, uint16_t v
   }
 
   if (extension == HSA_EXTENSION_AMD_LOADER) {
-    if (version_major > 1) return HSA_STATUS_ERROR;
-    hsa_ven_amd_loader_1_00_pfn_t ext_table;
+    if (version_major != 1) return HSA_STATUS_ERROR;
+    hsa_ven_amd_loader_1_01_pfn_t ext_table;
     ext_table.hsa_ven_amd_loader_query_host_address = hsa_ven_amd_loader_query_host_address;
     ext_table.hsa_ven_amd_loader_query_segment_descriptors =
         hsa_ven_amd_loader_query_segment_descriptors;
     ext_table.hsa_ven_amd_loader_query_executable = hsa_ven_amd_loader_query_executable;
+    ext_table.hsa_ven_amd_loader_executable_iterate_loaded_code_objects =
+        hsa_ven_amd_loader_executable_iterate_loaded_code_objects;
+    ext_table.hsa_ven_amd_loader_loaded_code_object_get_info =
+        hsa_ven_amd_loader_loaded_code_object_get_info;
 
     memcpy(table, &ext_table, Min(sizeof(ext_table), table_length));
 
@@ -487,7 +492,7 @@ hsa_status_t hsa_system_get_major_extension_table(uint16_t extension, uint16_t v
 //---------------------------------------------------------------------------//
 //  Agent
 //---------------------------------------------------------------------------//
-hsa_status_t 
+hsa_status_t
     hsa_iterate_agents(hsa_status_t (*callback)(hsa_agent_t agent, void* data),
                        void* data) {
   TRY;
@@ -546,7 +551,7 @@ hsa_status_t hsa_agent_iterate_caches(hsa_agent_t agent_handle,
   CATCH;
 }
 
-hsa_status_t 
+hsa_status_t
     hsa_agent_extension_supported(uint16_t extension, hsa_agent_t agent_handle,
                                   uint16_t version_major,
                                   uint16_t version_minor, bool* result) {
@@ -1036,7 +1041,7 @@ hsa_status_t hsa_memory_deregister(void* address, size_t size) {
   CATCH;
 }
 
-hsa_status_t 
+hsa_status_t
     hsa_memory_allocate(hsa_region_t region, size_t size, void** ptr) {
   TRY;
   IS_OPEN();
@@ -1103,7 +1108,7 @@ hsa_status_t hsa_memory_copy(void* dst, const void* src, size_t size) {
 // Signals
 //-----------------------------------------------------------------------------
 
-hsa_status_t 
+hsa_status_t
     hsa_signal_create(hsa_signal_value_t initial_value, uint32_t num_consumers,
                       const hsa_agent_t* consumers, hsa_signal_t* hsa_signal) {
   return AMD::hsa_amd_signal_create(initial_value, num_consumers, consumers, 0, hsa_signal);
@@ -1151,7 +1156,7 @@ void hsa_signal_store_screlease(hsa_signal_t hsa_signal, hsa_signal_value_t valu
   CATCHRET(void);
 }
 
-hsa_signal_value_t 
+hsa_signal_value_t
     hsa_signal_wait_relaxed(hsa_signal_t hsa_signal,
                             hsa_signal_condition_t condition,
                             hsa_signal_value_t compare_value,
@@ -1400,7 +1405,7 @@ void hsa_signal_subtract_scacq_screl(hsa_signal_t hsa_signal, hsa_signal_value_t
   CATCHRET(void);
 }
 
-hsa_signal_value_t 
+hsa_signal_value_t
     hsa_signal_exchange_relaxed(hsa_signal_t hsa_signal,
                                 hsa_signal_value_t value) {
   TRY;
@@ -1829,13 +1834,13 @@ hsa_status_t hsa_code_object_get_info(
       if (status != HSA_STATUS_SUCCESS) {
         return status;
       }
-  
+
       hsa_isa_t isa_handle = {0};
       status = HSA::hsa_isa_from_name(isa_name, &isa_handle);
       if (status != HSA_STATUS_SUCCESS) {
         return status;
       }
-  
+
       *((hsa_isa_t*)value) = isa_handle;
       return HSA_STATUS_SUCCESS;
     }
@@ -2541,6 +2546,9 @@ hsa_status_t hsa_status_string(
     case HSA_STATUS_ERROR_INVALID_ISA:
       *status_string = "HSA_STATUS_ERROR_INVALID_ISA: The instruction set architecture is invalid.";
       break;
+    case HSA_STATUS_ERROR_INVALID_ISA_NAME:
+      *status_string = "HSA_STATUS_ERROR_INVALID_ISA_NAME: The instruction set architecture name is invalid.";
+      break;
     case HSA_STATUS_ERROR_INVALID_CODE_OBJECT:
       *status_string = "HSA_STATUS_ERROR_INVALID_CODE_OBJECT: The code object is invalid.";
       break;

runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp

@@ -2,24 +2,24 @@
 //
 // The University of Illinois/NCSA
 // Open Source License (NCSA)
-// 
+//
 // Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
-// 
+//
 // Developed by:
-// 
+//
 //                 AMD Research and AMD HSA Software Development
-// 
+//
 //                 Advanced Micro Devices, Inc.
-// 
+//
 //                 www.amd.com
-// 
+//
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to
 // deal with the Software without restriction, including without limitation
 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
 // and/or sell copies of the Software, and to permit persons to whom the
 // Software is furnished to do so, subject to the following conditions:
-// 
+//
 //  - Redistributions of source code must retain the above copyright notice,
 //    this list of conditions and the following disclaimers.
 //  - Redistributions in binary form must reproduce the above copyright
@@ -29,7 +29,7 @@
 //    nor the names of its contributors may be used to endorse or promote
 //    products derived from this Software without specific prior written
 //    permission.
-// 
+//
 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
@@ -73,7 +73,7 @@ void HsaApiTable::Init() {
   // Update Api table for Amd Extensions and its major id
   UpdateAmdExts();
   hsa_api.amd_ext_ = &amd_ext_api;
- 
+
   // Initialize Api tables for Finalizer, Image, AqlProfile to NULL
   // The tables are initialized as part
   // of Hsa Runtime initialization, including their major ids
@@ -87,7 +87,7 @@ void HsaApiTable::Reset() {
 }
 
 void HsaApiTable::CloneExts(void* ext_table, uint32_t table_id) {
-  
+
   assert(ext_table != NULL && "Invalid extension table linked.");
 
   // Update HSA Extension Finalizer Api table
@@ -113,20 +113,20 @@ void HsaApiTable::CloneExts(void* ext_table, uint32_t table_id) {
 }
 
 void HsaApiTable::LinkExts(void* ext_table, uint32_t table_id) {
-  
+
   assert(ext_table != NULL && "Invalid extension table linked.");
 
   // Update HSA Extension Finalizer Api table
   if (table_id == HSA_EXT_FINALIZER_API_TABLE_ID) {
     finalizer_api = (*(FinalizerExtTable *)ext_table);
-    hsa_api.finalizer_ext_ = (FinalizerExtTable *)ext_table; 
+    hsa_api.finalizer_ext_ = (FinalizerExtTable *)ext_table;
     return;
   }
 
   // Update HSA Extension Image Api table
   if (table_id == HSA_EXT_IMAGE_API_TABLE_ID) {
     image_api = (*(ImageExtTable *)ext_table);
-    hsa_api.image_ext_ = (ImageExtTable *)ext_table; 
+    hsa_api.image_ext_ = (ImageExtTable *)ext_table;
     return;
   }
 
@@ -140,7 +140,7 @@ void HsaApiTable::LinkExts(void* ext_table, uint32_t table_id) {
 
 // Update Api table for Hsa Core Runtime
 void HsaApiTable::UpdateCore() {
-  
+
   // Initialize Version of Api Table
   core_api.version.major_id = HSA_CORE_API_TABLE_MAJOR_VERSION;
   core_api.version.minor_id = sizeof(::CoreApiTable);
@@ -341,12 +341,12 @@ void HsaApiTable::UpdateCore() {
 // member variable hsa_amd_image_create_fn while loading
 // Image extension library
 void HsaApiTable::UpdateAmdExts() {
-  
+
   // Initialize Version of Api Table
   amd_ext_api.version.major_id = HSA_AMD_EXT_API_TABLE_MAJOR_VERSION;
   amd_ext_api.version.minor_id = sizeof(::AmdExtTable);
   amd_ext_api.version.step_id = HSA_AMD_EXT_API_TABLE_STEP_VERSION;
-  
+
   // Initialize function pointers for Amd Extension Api's
   amd_ext_api.hsa_amd_coherency_get_type_fn = AMD::hsa_amd_coherency_get_type;
   amd_ext_api.hsa_amd_coherency_set_type_fn = AMD::hsa_amd_coherency_set_type;
@@ -381,6 +381,7 @@ void HsaApiTable::UpdateAmdExts() {
   amd_ext_api.hsa_amd_signal_create_fn = AMD::hsa_amd_signal_create;
   amd_ext_api.hsa_amd_ipc_signal_create_fn = AMD::hsa_amd_ipc_signal_create;
   amd_ext_api.hsa_amd_ipc_signal_attach_fn = AMD::hsa_amd_ipc_signal_attach;
+  amd_ext_api.hsa_amd_register_system_event_handler_fn = AMD::hsa_amd_register_system_event_handler;
 }
 
 class Init {

runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -798,4 +798,19 @@ hsa_status_t hsa_amd_ipc_signal_attach(const hsa_amd_ipc_signal_t* handle,
   CATCH;
 }
 
+hsa_status_t hsa_amd_register_system_event_handler(
+    hsa_amd_event_t type,
+    hsa_status_t (*callback)(const void* event_specific_data, void* data),
+    void* data) {
+  TRY;
+  IS_OPEN();
+  switch (type) {
+    case GPU_MEMORY_FAULT_EVENT:
+      return core::Runtime::runtime_singleton_->SetCustomVMFaultHandler(callback, data);
+    default:
+      return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+  CATCH;
+}
+
 } // end of AMD namespace

runtime/hsa-runtime/core/runtime/hsa_ven_amd_loader.cpp

@@ -45,9 +45,13 @@
 #include "core/inc/amd_hsa_loader.hpp"
 #include "core/inc/runtime.h"
 
+using namespace amd::hsa;
 using namespace core;
 
-hsa_status_t HSA_API hsa_ven_amd_loader_query_host_address(
+using loader::Executable;
+using loader::LoadedCodeObject;
+
+hsa_status_t hsa_ven_amd_loader_query_host_address(
   const void *device_address,
   const void **host_address) {
   if (false == core::Runtime::runtime_singleton_->IsOpen()) {
@@ -70,7 +74,7 @@ hsa_status_t HSA_API hsa_ven_amd_loader_query_host_address(
   return HSA_STATUS_SUCCESS;
 }
 
-hsa_status_t HSA_API hsa_ven_amd_loader_query_segment_descriptors(
+hsa_status_t hsa_ven_amd_loader_query_segment_descriptors(
   hsa_ven_amd_loader_segment_descriptor_t *segment_descriptors,
   size_t *num_segment_descriptors) {
   if (false == core::Runtime::runtime_singleton_->IsOpen()) {
@@ -81,7 +85,7 @@ hsa_status_t HSA_API hsa_ven_amd_loader_query_segment_descriptors(
   return Runtime::runtime_singleton_->loader()->QuerySegmentDescriptors(segment_descriptors, num_segment_descriptors);
 }
 
-hsa_status_t HSA_API hsa_ven_amd_loader_query_executable(
+hsa_status_t hsa_ven_amd_loader_query_executable(
   const void *device_address,
   hsa_executable_t *executable) {
 
@@ -101,3 +105,110 @@ hsa_status_t HSA_API hsa_ven_amd_loader_query_executable(
   *executable = exec;
   return HSA_STATUS_SUCCESS;
 }
+
+hsa_status_t hsa_ven_amd_loader_executable_iterate_loaded_code_objects(
+  hsa_executable_t executable,
+  hsa_status_t (*callback)(
+    hsa_executable_t executable,
+    hsa_loaded_code_object_t loaded_code_object,
+    void *data),
+  void *data) {
+  if (false == core::Runtime::runtime_singleton_->IsOpen()) {
+    return HSA_STATUS_ERROR_NOT_INITIALIZED;
+  }
+  if (nullptr == callback) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  Executable *exec = Executable::Object(executable);
+  if (!exec) {
+    return HSA_STATUS_ERROR_INVALID_EXECUTABLE;
+  }
+
+  return exec->IterateLoadedCodeObjects(callback, data);
+}
+
+hsa_status_t hsa_ven_amd_loader_loaded_code_object_get_info(
+  hsa_loaded_code_object_t loaded_code_object,
+  hsa_ven_amd_loader_loaded_code_object_info_t attribute,
+  void *value) {
+  if (false == core::Runtime::runtime_singleton_->IsOpen()) {
+    return HSA_STATUS_ERROR_NOT_INITIALIZED;
+  }
+  if (nullptr == value) {
+    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+  }
+
+  const LoadedCodeObject *lcobj = LoadedCodeObject::Object(loaded_code_object);
+  if (!lcobj) {
+    return HSA_STATUS_ERROR_INVALID_CODE_OBJECT;
+  }
+
+  switch (attribute) {
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_EXECUTABLE: {
+      *((hsa_executable_t*)value) = lcobj->getExecutable();
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_KIND: {
+      *((uint32_t*)value) = lcobj->getAgent().handle == 0
+           ? HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_KIND_PROGRAM
+           : HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_KIND_AGENT;
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_AGENT: {
+      hsa_agent_t agent = lcobj->getAgent();
+      if (agent.handle == 0) {
+          return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+      }
+      *((hsa_agent_t*)value) = agent;
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_CODE_OBJECT_STORAGE_TYPE: {
+      // TODO Update loader so it keeps track if code object was loaded from a
+      // file or memory.
+      *((uint32_t*)value) = HSA_VEN_AMD_LOADER_CODE_OBJECT_STORAGE_TYPE_MEMORY;
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_CODE_OBJECT_STORAGE_MEMORY_BASE: {
+      *((uint64_t*)value) = lcobj->getElfData();
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_CODE_OBJECT_STORAGE_MEMORY_SIZE: {
+      *((uint64_t*)value) = lcobj->getElfSize();
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_CODE_OBJECT_STORAGE_FILE: {
+      // TODO Update loader so it keeps track if code object was loaded from a
+      // file or memory.
+      return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_LOAD_DELTA: {
+      // TODO Check if executable is frozen.
+      // This suggests this code should be moved into LoadedCodeObjectImpl::getinfo
+      // as is done for other *_get_info methods. Currently LoadedCodeObject has a
+      // GetInfo method which is likely not used.
+      // Also should this have a *NOT_FROZEN ststus code added?
+      // if (state_ != HSA_EXECUTABLE_STATE_FROZEN) {
+      //   return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+      // }
+      *((int64_t*)value) = lcobj->getDelta();
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_LOAD_BASE: {
+      // TODO Check if executable is frozen.
+      *((uint64_t*)value) = lcobj->getLoadBase();
+      break;
+    }
+    case HSA_VEN_AMD_LOADER_LOADED_CODE_OBJECT_INFO_LOAD_SIZE: {
+      // TODO Check if executable is frozen.
+      *((uint64_t*)value) = lcobj->getLoadSize();
+      break;
+    }
+    default: {
+      return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+    }
+  }
+
+  return HSA_STATUS_SUCCESS;
+}

runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -589,8 +589,6 @@ hsa_status_t Runtime::GetSystemInfo(hsa_system_info_t attribute, void* value) {
   return HSA_STATUS_SUCCESS;
 }
 
-uint32_t Runtime::GetQueueId() { return atomic::Increment(&queue_count_); }
-
 hsa_status_t Runtime::SetAsyncSignalHandler(hsa_signal_t signal,
                                             hsa_signal_condition_t cond,
                                             hsa_signal_value_t value,
@@ -1044,48 +1042,86 @@ bool Runtime::VMFaultHandler(hsa_signal_value_t val, void* arg) {
     return false;
   }
 
-  if (runtime_singleton_->flag().enable_vm_fault_message()) {
-    HsaEvent* vm_fault_event = vm_fault_signal->EopEvent();
+  HsaEvent* vm_fault_event = vm_fault_signal->EopEvent();
 
-    const HsaMemoryAccessFault& fault =
-        vm_fault_event->EventData.EventData.MemoryAccessFault;
+  HsaMemoryAccessFault& fault =
+      vm_fault_event->EventData.EventData.MemoryAccessFault;
 
-    std::string reason = "";
+  hsa_status_t custom_handler_status = HSA_STATUS_ERROR;
+  // If custom handler is registered, pack the fault info and call the handler
+  if (runtime_singleton_->vm_fault_handler_custom_ != nullptr) {
+    hsa_amd_gpu_memory_fault_info_t* fault_info = new hsa_amd_gpu_memory_fault_info_t;
+
+    // Find the faulty agent
+    auto it = runtime_singleton_->agents_by_node_.find(fault.NodeId);
+    assert(it != runtime_singleton_->agents_by_node_.end() && "Can't find faulty agent.");
+    Agent* faulty_agent = it->second.front();
+    fault_info->agent = Agent::Convert(faulty_agent);
+
+    fault_info->virtual_address = fault.VirtualAddress;
+    fault_info->fault_reason_mask = 0x00000000;
     if (fault.Failure.NotPresent == 1) {
-      reason += "Page not present or supervisor privilege";
-    } else if (fault.Failure.ReadOnly == 1) {
-      reason += "Write access to a read-only page";
-    } else if (fault.Failure.NoExecute == 1) {
-      reason += "Execute access to a page marked NX";
-    } else if (fault.Failure.GpuAccess == 1) {
-      reason += "Host access only";
-    } else if (fault.Failure.ECC == 1) {
-      reason += "ECC failure (if supported by HW)";
-    } else {
-      reason += "Unknown";
+      fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00000001;
+    }
+    if (fault.Failure.ReadOnly == 1) {
+      fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00000010;
+    }
+    if (fault.Failure.NoExecute == 1) {
+      fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00000100;
+    }
+    if (fault.Failure.GpuAccess == 1) {
+      fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00001000;
+    }
+    if (fault.Failure.ECC == 1) {
+      fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00010000;
+    }
+    if (fault.Failure.Imprecise == 1) {
+      fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00100000;
     }
 
-    fprintf(stderr,
-            "Memory access fault by GPU node-%u on address %p%s. Reason: %s.\n",
-            fault.NodeId, reinterpret_cast<const void*>(fault.VirtualAddress),
-            (fault.Failure.Imprecise == 1) ? "(may not be exact address)" : "",
-            reason.c_str());
-  } else {
-    assert(false && "GPU memory access fault.");
+    custom_handler_status = runtime_singleton_->vm_fault_handler_custom_(fault_info,
+        runtime_singleton_->vm_fault_handler_user_data_);
   }
 
-  std::abort();
+  // No custom VM fault handler registered or it failed.
+  if (custom_handler_status != HSA_STATUS_SUCCESS) {
+    if (runtime_singleton_->flag().enable_vm_fault_message()) {
+      std::string reason = "";
+      if (fault.Failure.NotPresent == 1) {
+        reason += "Page not present or supervisor privilege";
+      } else if (fault.Failure.ReadOnly == 1) {
+        reason += "Write access to a read-only page";
+      } else if (fault.Failure.NoExecute == 1) {
+        reason += "Execute access to a page marked NX";
+      } else if (fault.Failure.GpuAccess == 1) {
+        reason += "Host access only";
+      } else if (fault.Failure.ECC == 1) {
+        reason += "ECC failure (if supported by HW)";
+      } else {
+        reason += "Unknown";
+      }
 
+      fprintf(stderr,
+              "Memory access fault by GPU node-%u on address %p%s. Reason: %s.\n",
+              fault.NodeId, reinterpret_cast<const void*>(fault.VirtualAddress),
+              (fault.Failure.Imprecise == 1) ? "(may not be exact address)" : "",
+              reason.c_str());
+    } else {
+      assert(false && "GPU memory access fault.");
+    }
+
+    std::abort();
+  }
   // No need to keep the signal because we are done.
   return false;
 }
 
 Runtime::Runtime()
     : blit_agent_(NULL),
-      queue_count_(0),
       sys_clock_freq_(0),
       vm_fault_event_(nullptr),
       vm_fault_signal_(nullptr),
+      vm_fault_handler_custom_(nullptr),
       ref_count_(0) {
   start_svm_address_ = 0;
 #if defined(HSA_LARGE_MODEL)
@@ -1363,4 +1399,15 @@ void Runtime::AsyncEvents::Clear() {
   arg_.clear();
 }
 
+hsa_status_t Runtime::SetCustomVMFaultHandler(
+    hsa_status_t (*callback)(const void* event_specific_data, void* data),
+    void* data) {
+  if (vm_fault_handler_custom_ != nullptr) {
+    return HSA_STATUS_ERROR;
+  } else {
+    vm_fault_handler_custom_ = callback;
+    vm_fault_handler_user_data_ = data;
+    return HSA_STATUS_SUCCESS;
+  }
+}
 }  // namespace core