Enable D2D SDMA Ganging over xGMI

Use all available SDMA engines capped by xGMI bandwith for all D2D copies within a hive. By default, set the latency boundary copy size as 4KB and below. Any copy size in within this boundary will not gang. Avoid oversubscribing engines by not ganging on engines with pending non-ganged work. An enviroment variable HSA_ENABLE_SDMA_GANG has been provided to override default ganging behaviour. Change-Id: Iccde76aa1af1d47ea2a151789432c9db4f0ffa8d [ROCm/ROCR-Runtime commit: 7df0167821]
2022-11-21 10:34:43 -05:00
@@ -95,10 +95,11 @@ class BlitKernel : public core::Blit {
  /// @param size Size of the data to be copied.
  /// @param dep_signals Arrays of dependent signal.
  /// @param out_signal Output signal.
+  /// @param gang_signals Array of gang signals.
  virtual hsa_status_t SubmitLinearCopyCommand(
      void* dst, const void* src, size_t size,
      std::vector<core::Signal*>& dep_signals,
-      core::Signal& out_signal) override;
+      core::Signal& out_signal, std::vector<core::Signal*>& gang_signals) override;

  /// @brief Submit an AQL packet to perform memory fill. The call is blocking
  /// until the command execution is finished.
@@ -113,6 +114,11 @@ class BlitKernel : public core::Blit {

  virtual uint64_t PendingBytes() override;

+  void GangLeader(bool gang_leader) {}
+  bool GangLeader() const { return false; }
+  void GangStatus(bool is_ganged) {}
+  bool GangStatus() const { return false; }
+
 private:
  union KernelArgs {
    struct __ALIGNED__(16) {
@@ -121,10 +121,11 @@ class BlitSdma : public BlitSdmaBase {
  /// @param size Size of the data to be copied.
  /// @param dep_signals Arrays of dependent signal.
  /// @param out_signal Output signal.
+  /// @param gang_signals Array of gang signals.
  virtual hsa_status_t SubmitLinearCopyCommand(
      void* dst, const void* src, size_t size,
      std::vector<core::Signal*>& dep_signals,
-      core::Signal& out_signal) override;
+      core::Signal& out_signal, std::vector<core::Signal*>& gang_signals) override;

  virtual hsa_status_t SubmitCopyRectCommand(const hsa_pitched_ptr_t* dst,
                                             const hsa_dim3_t* dst_offset,
@@ -144,6 +145,10 @@ class BlitSdma : public BlitSdmaBase {
  virtual hsa_status_t EnableProfiling(bool enable) override;

  virtual uint64_t PendingBytes() override;
+  void GangLeader(bool gang_leader) { gang_leader_ = gang_leader; }
+  bool GangLeader() const { return gang_leader_; }
+  void GangStatus(bool is_ganged) { is_ganged_ = is_ganged; }
+  bool GangStatus() const { return is_ganged_; }

 private:
  /// @brief Acquires the address into queue buffer where a new command
@@ -216,7 +221,7 @@ class BlitSdma : public BlitSdmaBase {

  hsa_status_t SubmitCommand(const void* cmds, size_t cmd_size, uint64_t size,
                             const std::vector<core::Signal*>& dep_signals,
-                             core::Signal& out_signal);
+                             core::Signal& out_signal, std::vector<core::Signal*>& gang_signals);

  hsa_status_t SubmitBlockingCommand(const void* cmds, size_t cmd_size, uint64_t size);

@@ -300,6 +305,12 @@ class BlitSdma : public BlitSdmaBase {

  /// True if sDMA supports HDP flush
  bool hdp_flush_support_;
+
+  /// True if SDMA blit is gang leader
+  bool gang_leader_;
+
+  /// True if SDMA blit is ganged
+  bool is_ganged_;
 };

 // Ring indices are 32-bit.
@@ -145,6 +145,8 @@ class GpuAgentInt : public core::Agent {
  // @retval Coherency type.
  virtual hsa_amd_coherency_type_t current_coherency_type() const = 0;

+  virtual void RegisterGangPeer(core::Agent& gang_peer, unsigned int bandwidth_factor) = 0;
+
  // @brief Query if agent represent Kaveri GPU.
  //
  // @retval true if agent is Kaveri GPU.
@@ -290,6 +292,8 @@ class GpuAgent : public GpuAgentInt {
  }

  core::Agent* GetNearestCpuAgent(void) const;
+  
+  void RegisterGangPeer(core::Agent& gang_peer, unsigned int bandwidth_factor) override;

  // Getter & setters.

@@ -509,7 +513,6 @@ class GpuAgent : public GpuAgentInt {

  // @brief HDP flush registers
  hsa_amd_hdp_flush_t HDP_flush_ = {nullptr, nullptr};
-
 private:
  // @brief Query the driver to get the region list owned by this agent.
  void InitRegionList();
@@ -546,7 +549,7 @@ class GpuAgent : public GpuAgentInt {
  void ReleaseScratch(void* base, size_t size, bool large);

  // Bind index of peer device that is connected via xGMI links
-  lazy_ptr<core::Blit>& GetXgmiBlit(const core::Agent& peer_agent);
+  lazy_ptr<core::Blit>& GetXgmiBlit(const core::Agent& peer_agent, int gang_id);

  // Bind the Blit object that will drive the copy operation
  // across PCIe links (H2D or D2H) or is within same device D2D
@@ -554,7 +557,7 @@ class GpuAgent : public GpuAgentInt {

  // Bind the Blit object that will drive the copy operation
  lazy_ptr<core::Blit>& GetBlitObject(const core::Agent& dst_agent, const core::Agent& src_agent,
-                                      const size_t size);
+                                      const size_t size, int gang_id);

  // Bind the Blit object that will drive the copy operation by engine ID
  lazy_ptr<core::Blit>& GetBlitObject(uint32_t engine_id);
@@ -599,6 +602,8 @@ class GpuAgent : public GpuAgentInt {

  // Check if SDMA engine by ID is free
  bool DmaEngineIsFree(uint32_t engine_id);
+
+  std::vector<std::pair<core::Agent&,unsigned int>> gang_peers_info_;
 };

 }  // namespace amd
@@ -85,9 +85,11 @@ class Blit {
  /// @param size Size of the data to be copied.
  /// @param dep_signals Arrays of dependent signal.
  /// @param out_signal Output signal.
+  /// @param gang_signals Array of gang signals.
  virtual hsa_status_t SubmitLinearCopyCommand(
      void* dst, const void* src, size_t size,
-      std::vector<core::Signal*>& dep_signals, core::Signal& out_signal) = 0;
+      std::vector<core::Signal*>& dep_signals, core::Signal& out_signal,
+      std::vector<core::Signal*>& gang_signals) = 0;

  /// @brief Submit a linear fill command to the the underlying compute device's
  /// control block. The call is blocking until the command execution is
@@ -114,6 +116,11 @@ class Blit {
  /// @Brief Reports the approximate number of remaining bytes to copy or fill.  Any return of zero
  /// must be exact.
  virtual uint64_t PendingBytes() = 0;
+
+  virtual void GangLeader(bool gang_leader) = 0;
+  virtual bool GangLeader() const { return false; };
+  virtual void GangStatus(bool is_ganged) = 0;
+  virtual bool GangStatus() const { return false; };
 };
 }  // namespace core
 }  // namespace rocr