SWDEV-523281 - [clr] Implementation of hipLaunchKernelExC and hipDrvLaunchKernelEx API with support for cooperative launch (#92)

hipamd/include/hip/amd_detail/hip_api_trace.hpp

@@ -63,7 +63,7 @@
 #define HIP_API_TABLE_STEP_VERSION 0
 #define HIP_COMPILER_API_TABLE_STEP_VERSION 0
 #define HIP_TOOLS_API_TABLE_STEP_VERSION 0
-#define HIP_RUNTIME_API_TABLE_STEP_VERSION 9
+#define HIP_RUNTIME_API_TABLE_STEP_VERSION 11
 
 // HIP API interface
 // HIP compiler dispatch functions
@@ -1036,6 +1036,10 @@ typedef hipError_t (*t_hipGraphBatchMemOpNodeSetParams)(hipGraphNode_t hNode,
 typedef hipError_t (*t_hipGraphExecBatchMemOpNodeSetParams)(
     hipGraphExec_t hGraphExec, hipGraphNode_t hNode, const hipBatchMemOpNodeParams* nodeParams);
 typedef hipError_t (*t_hipEventRecordWithFlags)(hipEvent_t event, hipStream_t stream, unsigned int flags);
+typedef hipError_t (*t_hipLaunchKernelExC)(const hipLaunchConfig_t* config, const void* fPtr,
+                                           void** args);
+typedef hipError_t (*t_hipDrvLaunchKernelEx)(const HIP_LAUNCH_CONFIG* config, hipFunction_t f,
+                                             void** params, void** extra);
 
 // HIP Compiler dispatch table
 struct HipCompilerDispatchTable {
@@ -1571,6 +1575,10 @@ struct HipDispatchTable {
   t_hipLinkCreate hipLinkCreate_fn;
   t_hipLinkDestroy hipLinkDestroy_fn;
 
+  // HIP_RUNTIME_API_TABLE_STEP_VERSION = 11
+  t_hipLaunchKernelExC hipLaunchKernelExC_fn;
+  t_hipDrvLaunchKernelEx hipDrvLaunchKernelEx_fn;
+
   // DO NOT EDIT ABOVE!
   // HIP_RUNTIME_API_TABLE_STEP_VERSION == 11
 

hipamd/include/hip/amd_detail/hip_prof_str.h

@@ -436,7 +436,9 @@ enum hip_api_id_t {
   HIP_API_ID_hipLinkComplete = 416,
   HIP_API_ID_hipLinkCreate = 417,
   HIP_API_ID_hipLinkDestroy = 418,
-  HIP_API_ID_LAST = 418,
+  HIP_API_ID_hipLaunchKernelExC = 419,
+  HIP_API_ID_hipDrvLaunchKernelEx = 420,
+  HIP_API_ID_LAST = 420,
 
   HIP_API_ID_hipChooseDevice = HIP_API_ID_CONCAT(HIP_API_ID_,hipChooseDevice),
   HIP_API_ID_hipGetDeviceProperties = HIP_API_ID_CONCAT(HIP_API_ID_,hipGetDeviceProperties),
@@ -545,6 +547,7 @@ static inline const char* hip_api_name(const uint32_t id) {
     case HIP_API_ID_hipDrvGraphExecMemsetNodeSetParams: return "hipDrvGraphExecMemsetNodeSetParams";
     case HIP_API_ID_hipDrvGraphMemcpyNodeGetParams: return "hipDrvGraphMemcpyNodeGetParams";
     case HIP_API_ID_hipDrvGraphMemcpyNodeSetParams: return "hipDrvGraphMemcpyNodeSetParams";
+    case HIP_API_ID_hipDrvLaunchKernelEx: return "hipDrvLaunchKernelEx";
     case HIP_API_ID_hipDrvMemcpy2DUnaligned: return "hipDrvMemcpy2DUnaligned";
     case HIP_API_ID_hipDrvMemcpy3D: return "hipDrvMemcpy3D";
     case HIP_API_ID_hipDrvMemcpy3DAsync: return "hipDrvMemcpy3DAsync";
@@ -704,6 +707,7 @@ static inline const char* hip_api_name(const uint32_t id) {
     case HIP_API_ID_hipLaunchCooperativeKernelMultiDevice: return "hipLaunchCooperativeKernelMultiDevice";
     case HIP_API_ID_hipLaunchHostFunc: return "hipLaunchHostFunc";
     case HIP_API_ID_hipLaunchKernel: return "hipLaunchKernel";
+    case HIP_API_ID_hipLaunchKernelExC: return "hipLaunchKernelExC";
     case HIP_API_ID_hipLinkAddData: return "hipLinkAddData";
     case HIP_API_ID_hipLinkAddFile: return "hipLinkAddFile";
     case HIP_API_ID_hipLinkComplete: return "hipLinkComplete";
@@ -958,6 +962,7 @@ static inline uint32_t hipApiIdByName(const char* name) {
   if (strcmp("hipDrvGraphExecMemsetNodeSetParams", name) == 0) return HIP_API_ID_hipDrvGraphExecMemsetNodeSetParams;
   if (strcmp("hipDrvGraphMemcpyNodeGetParams", name) == 0) return HIP_API_ID_hipDrvGraphMemcpyNodeGetParams;
   if (strcmp("hipDrvGraphMemcpyNodeSetParams", name) == 0) return HIP_API_ID_hipDrvGraphMemcpyNodeSetParams;
+  if (strcmp("hipDrvLaunchKernelEx", name) == 0) return HIP_API_ID_hipDrvLaunchKernelEx;
   if (strcmp("hipDrvMemcpy2DUnaligned", name) == 0) return HIP_API_ID_hipDrvMemcpy2DUnaligned;
   if (strcmp("hipDrvMemcpy3D", name) == 0) return HIP_API_ID_hipDrvMemcpy3D;
   if (strcmp("hipDrvMemcpy3DAsync", name) == 0) return HIP_API_ID_hipDrvMemcpy3DAsync;
@@ -1117,6 +1122,7 @@ static inline uint32_t hipApiIdByName(const char* name) {
   if (strcmp("hipLaunchCooperativeKernelMultiDevice", name) == 0) return HIP_API_ID_hipLaunchCooperativeKernelMultiDevice;
   if (strcmp("hipLaunchHostFunc", name) == 0) return HIP_API_ID_hipLaunchHostFunc;
   if (strcmp("hipLaunchKernel", name) == 0) return HIP_API_ID_hipLaunchKernel;
+  if (strcmp("hipLaunchKernelExC", name) == 0) return HIP_API_ID_hipLaunchKernelExC;
   if (strcmp("hipLinkAddData", name) == 0) return HIP_API_ID_hipLinkAddData;
   if (strcmp("hipLinkAddFile", name) == 0) return HIP_API_ID_hipLinkAddFile;
   if (strcmp("hipLinkComplete", name) == 0) return HIP_API_ID_hipLinkComplete;
@@ -1644,6 +1650,15 @@ typedef struct hip_api_data_s {
       const HIP_MEMCPY3D* nodeParams;
       HIP_MEMCPY3D nodeParams__val;
     } hipDrvGraphMemcpyNodeSetParams;
+    struct {
+      const HIP_LAUNCH_CONFIG* config;
+      HIP_LAUNCH_CONFIG config__val;
+      hipFunction_t f;
+      void** params;
+      void* params__val;
+      void** extra;
+      void* extra__val;
+    } hipDrvLaunchKernelEx;
     struct {
       const hip_Memcpy2D* pCopy;
       hip_Memcpy2D pCopy__val;
@@ -2607,6 +2622,13 @@ typedef struct hip_api_data_s {
       size_t sharedMemBytes;
       hipStream_t stream;
     } hipLaunchKernel;
+    struct {
+      const hipLaunchConfig_t* config;
+      hipLaunchConfig_t config__val;
+      const void* fPtr;
+      void** args;
+      void* args__val;
+    } hipLaunchKernelExC;
     struct {
       hipLinkState_t state;
       hipJitInputType type;
@@ -4092,6 +4114,13 @@ typedef struct hip_api_data_s {
   cb_data.args.hipDrvGraphMemcpyNodeSetParams.hNode = (hipGraphNode_t)hNode; \
   cb_data.args.hipDrvGraphMemcpyNodeSetParams.nodeParams = (const HIP_MEMCPY3D*)nodeParams; \
 };
+// hipDrvLaunchKernelEx[('const HIP_LAUNCH_CONFIG*', 'config'), ('hipFunction_t', 'f'), ('void**', 'params'), ('void**', 'extra')]
+#define INIT_hipDrvLaunchKernelEx_CB_ARGS_DATA(cb_data) { \
+  cb_data.args.hipDrvLaunchKernelEx.config = (const HIP_LAUNCH_CONFIG*)config; \
+  cb_data.args.hipDrvLaunchKernelEx.f = (hipFunction_t)f; \
+  cb_data.args.hipDrvLaunchKernelEx.params = (void**)kernelParams; \
+  cb_data.args.hipDrvLaunchKernelEx.extra = (void**)extra; \
+};
 // hipDrvMemcpy2DUnaligned[('const hip_Memcpy2D*', 'pCopy')]
 #define INIT_hipDrvMemcpy2DUnaligned_CB_ARGS_DATA(cb_data) { \
   cb_data.args.hipDrvMemcpy2DUnaligned.pCopy = (const hip_Memcpy2D*)pCopy; \
@@ -5049,6 +5078,12 @@ typedef struct hip_api_data_s {
   cb_data.args.hipLaunchKernel.sharedMemBytes = (size_t)sharedMemBytes; \
   cb_data.args.hipLaunchKernel.stream = (hipStream_t)stream; \
 };
+// hipLaunchKernelExC[('const hipLaunchConfig_t*', 'config'), ('const void*', 'fPtr'), ('void**', 'args')]
+#define INIT_hipLaunchKernelExC_CB_ARGS_DATA(cb_data) { \
+  cb_data.args.hipLaunchKernelExC.config = (const hipLaunchConfig_t*)config; \
+  cb_data.args.hipLaunchKernelExC.fPtr = (const void*)fPtr; \
+  cb_data.args.hipLaunchKernelExC.args = (void**)args; \
+};
 // hipLinkAddData[('hipLinkState_t', 'state'), ('hipJitInputType', 'type'), ('void*', 'data'), ('size_t', 'size'), ('const char*', 'name'), ('unsigned int', 'numOptions'), ('hipJitOption*', 'options'), ('void**', 'optionValues')]
 #define INIT_hipLinkAddData_CB_ARGS_DATA(cb_data) { \
   cb_data.args.hipLinkAddData.state = (hipLinkState_t)hip_link_state; \
@@ -6535,6 +6570,12 @@ static inline void hipApiArgsInit(hip_api_id_t id, hip_api_data_t* data) {
     case HIP_API_ID_hipDrvGraphMemcpyNodeSetParams:
       if (data->args.hipDrvGraphMemcpyNodeSetParams.nodeParams) data->args.hipDrvGraphMemcpyNodeSetParams.nodeParams__val = *(data->args.hipDrvGraphMemcpyNodeSetParams.nodeParams);
       break;
+// hipDrvLaunchKernelEx[('const HIP_LAUNCH_CONFIG*', 'config'), ('hipFunction_t', 'f'), ('void**', 'params'), ('void**', 'extra')]
+    case HIP_API_ID_hipDrvLaunchKernelEx:
+      if (data->args.hipDrvLaunchKernelEx.config) data->args.hipDrvLaunchKernelEx.config__val = *(data->args.hipDrvLaunchKernelEx.config);
+      if (data->args.hipDrvLaunchKernelEx.params) data->args.hipDrvLaunchKernelEx.params__val = *(data->args.hipDrvLaunchKernelEx.params);
+      if (data->args.hipDrvLaunchKernelEx.extra) data->args.hipDrvLaunchKernelEx.extra__val = *(data->args.hipDrvLaunchKernelEx.extra);
+      break;
 // hipDrvMemcpy2DUnaligned[('const hip_Memcpy2D*', 'pCopy')]
     case HIP_API_ID_hipDrvMemcpy2DUnaligned:
       if (data->args.hipDrvMemcpy2DUnaligned.pCopy) data->args.hipDrvMemcpy2DUnaligned.pCopy__val = *(data->args.hipDrvMemcpy2DUnaligned.pCopy);
@@ -7177,6 +7218,11 @@ static inline void hipApiArgsInit(hip_api_id_t id, hip_api_data_t* data) {
     case HIP_API_ID_hipLaunchKernel:
       if (data->args.hipLaunchKernel.args) data->args.hipLaunchKernel.args__val = *(data->args.hipLaunchKernel.args);
       break;
+// hipLaunchKernelExC[('const hipLaunchConfig_t*', 'config'), ('const void*', 'fPtr'), ('void**', 'args')]
+    case HIP_API_ID_hipLaunchKernelExC:
+      if (data->args.hipLaunchKernelExC.config) data->args.hipLaunchKernelExC.config__val = *(data->args.hipLaunchKernelExC.config);
+      if (data->args.hipLaunchKernelExC.args) data->args.hipLaunchKernelExC.args__val = *(data->args.hipLaunchKernelExC.args);
+      break;
 // hipLinkAddData[('hipLinkState_t', 'state'), ('hipJitInputType', 'type'), ('void*', 'data'), ('size_t', 'size'), ('const char*', 'name'), ('unsigned int', 'numOptions'), ('hipJitOption*', 'options'), ('void**', 'optionValues')]
     case HIP_API_ID_hipLinkAddData:
       if (data->args.hipLinkAddData.name) data->args.hipLinkAddData.name__val = *(data->args.hipLinkAddData.name);
@@ -8359,6 +8405,17 @@ static inline const char* hipApiString(hip_api_id_t id, const hip_api_data_t* da
       else { oss << ", nodeParams="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDrvGraphMemcpyNodeSetParams.nodeParams__val); }
       oss << ")";
     break;
+    case HIP_API_ID_hipDrvLaunchKernelEx:
+      oss << "hipDrvLaunchKernelEx(";
+      if (data->args.hipDrvLaunchKernelEx.config == NULL) oss << "config=NULL";
+      else { oss << "config="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDrvLaunchKernelEx.config__val); }
+      oss << ", f="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDrvLaunchKernelEx.f);
+      if (data->args.hipDrvLaunchKernelEx.params == NULL) oss << ", params=NULL";
+      else { oss << ", params="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDrvLaunchKernelEx.params__val); }
+      if (data->args.hipDrvLaunchKernelEx.extra == NULL) oss << ", extra=NULL";
+      else { oss << ", extra="; roctracer::hip_support::detail::operator<<(oss, data->args.hipDrvLaunchKernelEx.extra__val); }
+      oss << ")";
+    break;
     case HIP_API_ID_hipDrvMemcpy2DUnaligned:
       oss << "hipDrvMemcpy2DUnaligned(";
       if (data->args.hipDrvMemcpy2DUnaligned.pCopy == NULL) oss << "pCopy=NULL";
@@ -9646,6 +9703,15 @@ static inline const char* hipApiString(hip_api_id_t id, const hip_api_data_t* da
       oss << ", stream="; roctracer::hip_support::detail::operator<<(oss, data->args.hipLaunchKernel.stream);
       oss << ")";
     break;
+    case HIP_API_ID_hipLaunchKernelExC:
+      oss << "hipLaunchKernelExC(";
+      if (data->args.hipLaunchKernelExC.config == NULL) oss << "config=NULL";
+      else { oss << "config="; roctracer::hip_support::detail::operator<<(oss, data->args.hipLaunchKernelExC.config__val); }
+      oss << ", fPtr="; roctracer::hip_support::detail::operator<<(oss, data->args.hipLaunchKernelExC.fPtr);
+      if (data->args.hipLaunchKernelExC.args == NULL) oss << ", args=NULL";
+      else { oss << ", args="; roctracer::hip_support::detail::operator<<(oss, data->args.hipLaunchKernelExC.args__val); }
+      oss << ")";
+    break;
     case HIP_API_ID_hipLinkAddData:
       oss << "hipLinkAddData(";
       oss << "state="; roctracer::hip_support::detail::operator<<(oss, data->args.hipLinkAddData.state);

hipamd/src/amdhip.def

@@ -489,4 +489,6 @@ hipLinkAddData
 hipLinkAddFile
 hipLinkComplete
 hipLinkCreate
-hipLinkDestroy
+hipLinkDestroy
+hipLaunchKernelExC
+hipDrvLaunchKernelEx

hipamd/src/hip_api_trace.cpp

@@ -823,6 +823,10 @@ hipError_t hipGraphBatchMemOpNodeSetParams(hipGraphNode_t hNode,
 hipError_t hipGraphExecBatchMemOpNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNode_t hNode,
                                                const hipBatchMemOpNodeParams* nodeParams);
 hipError_t hipEventRecordWithFlags(hipEvent_t event, hipStream_t stream, unsigned flags);
+hipError_t hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args);
+hipError_t hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f, void** params,
+                                void** extra);
+
 }  // namespace hip
 
 namespace hip {
@@ -1335,6 +1339,8 @@ void UpdateDispatchTable(HipDispatchTable* ptrDispatchTable) {
   ptrDispatchTable->hipGraphExecBatchMemOpNodeSetParams_fn =
       hip::hipGraphExecBatchMemOpNodeSetParams;
   ptrDispatchTable->hipEventRecordWithFlags_fn = hip::hipEventRecordWithFlags;
+  ptrDispatchTable->hipLaunchKernelExC_fn = hip::hipLaunchKernelExC;
+  ptrDispatchTable->hipDrvLaunchKernelEx_fn = hip::hipDrvLaunchKernelEx;
 }
 
 #if HIP_ROCPROFILER_REGISTER > 0
@@ -1972,15 +1978,18 @@ HIP_ENFORCE_ABI(HipDispatchTable, hipLinkAddFile_fn , 470)
 HIP_ENFORCE_ABI(HipDispatchTable, hipLinkComplete_fn , 471)
 HIP_ENFORCE_ABI(HipDispatchTable, hipLinkCreate_fn , 472)
 HIP_ENFORCE_ABI(HipDispatchTable, hipLinkDestroy_fn , 473)
+// HIP_RUNTIME_API_TABLE_STEP_VERSION == 11
+HIP_ENFORCE_ABI(HipDispatchTable, hipLaunchKernelExC_fn, 474);
+HIP_ENFORCE_ABI(HipDispatchTable, hipDrvLaunchKernelEx_fn, 475);
 // if HIP_ENFORCE_ABI entries are added for each new function pointer in the table, the number below
 // will be +1 of the number in the last HIP_ENFORCE_ABI line. E.g.:
 //
 //  HIP_ENFORCE_ABI(<table>, <functor>, 8)
 //
 //  HIP_ENFORCE_ABI_VERSIONING(<table>, 9) <- 8 + 1 = 9
-HIP_ENFORCE_ABI_VERSIONING(HipDispatchTable, 474)
+HIP_ENFORCE_ABI_VERSIONING(HipDispatchTable, 476)
 
-static_assert(HIP_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIP_RUNTIME_API_TABLE_STEP_VERSION == 9,
+static_assert(HIP_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIP_RUNTIME_API_TABLE_STEP_VERSION == 11,
               "If you get this error, add new HIP_ENFORCE_ABI(...) code for the new function "
               "pointers and then update this check so it is true");
 #endif

hipamd/src/hip_hcc.map.in

@@ -595,4 +595,12 @@ global:
     hipLinkDestroy;
 local:
     *;
-} hip_6.2;
+} hip_6.2;
+
+hip_6.5 {
+global:
+    hipLaunchKernelExC;
+    hipDrvLaunchKernelEx;
+local:
+    *;
+} hip_6.4;

hipamd/src/hip_module.cpp

@@ -1077,4 +1077,65 @@ hipError_t hipLinkDestroy(hipLinkState_t hip_link_state) {
   HIP_RETURN(hipSuccess);
 }
 
+hipError_t hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args) {
+  HIP_INIT_API(hipLaunchKernelExC, config, fPtr, args);
+
+  if (fPtr == nullptr || config == nullptr || config->numAttrs == 0) {
+    HIP_RETURN(hipErrorInvalidConfiguration);
+  }
+
+  for (size_t attr_idx = 0; attr_idx < config->numAttrs; ++attr_idx) {
+    hipLaunchAttribute& attr = config->attrs[attr_idx];
+    switch (attr.id) {
+      case hipLaunchAttributeCooperative:
+        if (attr.val.cooperative != 0) {
+          HIP_RETURN_DURATION(
+              hipLaunchCooperativeKernel_common(fPtr, config->gridDim, config->blockDim, args,
+                                                config->dynamicSmemBytes, config->stream));
+        }
+        break;
+      default:
+        LogPrintfError("Attribute %u not supported", attr.id);
+        break;
+    }
+  }
+  HIP_RETURN(hipErrorInvalidConfiguration);
+}
+
+hipError_t hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f,
+                                void** kernelParams, void** extra) {
+  HIP_INIT_API(hipDrvLaunchKernelEx, config, f, kernelParams, extra);
+
+  if (f == nullptr || config == nullptr || config->numAttrs == 0) {
+    HIP_RETURN(hipErrorInvalidConfiguration);
+  }
+
+  size_t globalWorkSizeX = static_cast<size_t>(config->gridDimX) * config->blockDimX;
+  size_t globalWorkSizeY = static_cast<size_t>(config->gridDimY) * config->blockDimY;
+  size_t globalWorkSizeZ = static_cast<size_t>(config->gridDimZ) * config->blockDimZ;
+  if (globalWorkSizeX > std::numeric_limits<uint32_t>::max() ||
+      globalWorkSizeY > std::numeric_limits<uint32_t>::max() ||
+      globalWorkSizeZ > std::numeric_limits<uint32_t>::max()) {
+    HIP_RETURN(hipErrorInvalidConfiguration);
+  }
+
+  for (size_t attr_idx = 0; attr_idx < config->numAttrs; ++attr_idx) {
+    hipLaunchAttribute& attr = config->attrs[attr_idx];
+    switch (attr.id) {
+      case hipLaunchAttributeCooperative:
+        if (attr.value.cooperative != 0) {
+          HIP_RETURN(ihipModuleLaunchKernel(
+              f, static_cast<uint32_t>(globalWorkSizeX), static_cast<uint32_t>(globalWorkSizeY),
+              static_cast<uint32_t>(globalWorkSizeZ), config->blockDimX, config->blockDimY,
+              config->blockDimZ, config->sharedMemBytes, config->hStream, kernelParams, nullptr,
+              nullptr, nullptr, 0, amd::NDRangeKernelCommand::CooperativeGroups));
+        }
+        break;
+      default:
+        LogPrintfError("Attribute %u not supported", attr.id);
+        break;
+    }
+  }
+  HIP_RETURN(hipErrorInvalidConfiguration)
+}
 }  // namespace hip

hipamd/src/hip_table_interface.cpp

@@ -1868,4 +1868,13 @@ hipError_t hipGraphExecBatchMemOpNodeSetParams(hipGraphExec_t hGraphExec,
 }
 hipError_t hipEventRecordWithFlags(hipEvent_t event, hipStream_t stream, unsigned int flags) {
   return hip::GetHipDispatchTable()->hipEventRecordWithFlags_fn(event, stream, flags);
+}
+
+hipError_t hipLaunchKernelExC(const hipLaunchConfig_t* config, const void* fPtr, void** args) {
+  return hip::GetHipDispatchTable()->hipLaunchKernelExC_fn(config, fPtr, args);
+}
+
+hipError_t hipDrvLaunchKernelEx(const HIP_LAUNCH_CONFIG* config, hipFunction_t f, void** kernel,
+                                void** extra) {
+  return hip::GetHipDispatchTable()->hipDrvLaunchKernelEx_fn(config, f, kernel, extra);
 }