Add hipDeviceAttributeExpertSchedMode (#2435)

* Add hipDeviceAttributeExpertSchedMode

---------

Co-authored-by: Stefan Sokolovic <stefan.sokolovic2@amd.com>

* Update hipDeviceAttributeExpertSchedMode unit test

* Move check to ROCr from thunk interface

* Revert unrelated whitespace changes

* Revert version bump

---------

Co-authored-by: Stefan Sokolovic <stefan.sokolovic2@amd.com>

projects/clr/hipamd/src/hip_device_runtime.cpp

@@ -457,6 +457,9 @@ hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device)
     case hipDeviceAttributeHostNumaId:
       *pi = static_cast<int>(g_devices[device]->devices()[0]->getPreferredNumaNode());
       break;
+    case hipDeviceAttributeExpertSchedMode:
+      *pi = static_cast<int>(g_devices[device]->devices()[0]->info().hasExpertSchedMode_);
+      break;
     default:
       HIP_RETURN(hipErrorInvalidValue);
   }

projects/clr/rocclr/device/device.hpp

@@ -667,6 +667,8 @@ struct Info : public amd::EmbeddedObject {
   size_t scratchLimitMax;  //! Maximum size of scratch limit of this device memory in bytes.
 
   uint32_t numberOfXccs_;  //! The number of XCC(s) on the device
+
+  bool hasExpertSchedMode_;  //! Device supports expert scheduling mode
 };
 
 //! Device settings

projects/clr/rocclr/device/rocm/rocdevice.cpp

@@ -1638,6 +1638,14 @@ bool Device::populateOCLDeviceConstants() {
     LogError("HSA_AMD_AGENT_INFO_PM4_EMULATION query failed.");
   }
 
+  info_.hasExpertSchedMode_ = false;
+  if (HSA_STATUS_SUCCESS !=
+      Hsa::agent_get_info(bkendDevice_,
+                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE),
+                          &info_.hasExpertSchedMode_)) {
+    LogWarning("HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE query failed.");
+  }
+
   ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Gfx Major/Minor/Stepping: %d/%d/%d", isa().versionMajor(),
           isa().versionMinor(), isa().versionStepping());
   ClPrint(amd::LOG_INFO, amd::LOG_INIT, "HMM support: %d, XNACK: %d, Direct host access: %d",

projects/hip-tests/catch/unit/device/hipGetDeviceAttribute.cc

@@ -179,6 +179,7 @@ TEST_CASE("Unit_hipGetDeviceAttribute_CheckAttrValues") {
   HIP_CHECK(
       test_hipDeviceGetAttribute(deviceId, hipDeviceAttributeManagedMemory, props.managedMemory));
   HIP_CHECK(test_hipDeviceGetAttribute(deviceId, hipDeviceAttributeNumberOfXccs));
+  HIP_CHECK(test_hipDeviceGetAttribute(deviceId, hipDeviceAttributeExpertSchedMode));
 #endif
 
   HIP_CHECK(test_hipDeviceGetAttribute(deviceId, hipDeviceAttributeMaxPitch, props.memPitch));
@@ -356,7 +357,7 @@ constexpr AttributeToStringMap<33> kCudaOnlyAttributes{
 #endif
 
 #if HT_AMD
-constexpr AttributeToStringMap<18> kAmdOnlyAttributes{{
+constexpr AttributeToStringMap<19> kAmdOnlyAttributes{{
     {hipDeviceAttributeClockInstructionRate, "hipDeviceAttributeClockInstructionRate"},
     {hipDeviceAttributeUnused3, "hipDeviceAttributeUnused3"},
     {hipDeviceAttributeMaxSharedMemoryPerMultiprocessor,
@@ -380,7 +381,8 @@ constexpr AttributeToStringMap<18> kAmdOnlyAttributes{{
     {hipDeviceAttributePhysicalMultiProcessorCount,
      "hipDeviceAttributePhysicalMultiProcessorCount"},
     {hipDeviceAttributeFineGrainSupport, "hipDeviceAttributeFineGrainSupport"},
-    {hipDeviceAttributeNumberOfXccs, "hipDeviceAttributeNumberOfXccs"}
+    {hipDeviceAttributeNumberOfXccs, "hipDeviceAttributeNumberOfXccs"},
+    {hipDeviceAttributeExpertSchedMode, "hipDeviceAttributeExpertSchedMode"}
     // {hipDeviceAttributeWallClockRate, "hipDeviceAttributeWallClockRate"}
 }};
 #endif

projects/hip/include/hip/hip_runtime_api.h

@@ -598,6 +598,8 @@ typedef enum hipDeviceAttribute_t {
                                                  ///< indirectly addressable) VGPRs per thread in
                                                  ///< DWORDs.
   hipDeviceAttributePciChipId,                   ///< GPU Manufacturer device id
+  hipDeviceAttributeExpertSchedMode,             ///< '1' if Device supports expert scheduling mode,
+                                                 ///< '0' otherwise.
 
   hipDeviceAttributeAmdSpecificEnd = 19999,
   hipDeviceAttributeVendorSpecificBegin = 20000,

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -1716,6 +1716,15 @@ hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const {
       static_cast<hsa_luid_t*>(value)->low = properties_.LuidLowPart;
       static_cast<hsa_luid_t*>(value)->high = properties_.LuidHighPart;
       break;
+    case HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE: {
+      // Requires KFD version >= 1.20 AND GFX major version >= 12
+      auto kfd_version = core::Runtime::runtime_singleton_->KfdVersion().version;
+      *((bool*)value) = (kfd_version.KernelInterfaceMajorVersion > 1 ||
+                         (kfd_version.KernelInterfaceMajorVersion == 1 &&
+                          kfd_version.KernelInterfaceMinorVersion >= 20)) &&
+                        properties_.EngineId.ui32.Major >= 12;
+      break;
+    }
     default:
       return HSA_STATUS_ERROR_INVALID_ARGUMENT;
       break;

projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -715,6 +715,10 @@ typedef enum hsa_amd_agent_info_s {
    * valid on Windows. The type of this attribute is LUID.
    */
   HSA_AMD_AGENT_INFO_LUID = 0xA11A,
+  /**
+   * The agent supports expert scheduling mode. The type of this attribute is bool.
+   */
+  HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE = 0xA11B,
 } hsa_amd_agent_info_t;
 
 /**