rocm-systems/runtime/hsa-amd-aqlprofile/src/commandwriter/cmdwriter.h

// cmdwriter.h
// Header file for CommandWriter and CmdBuf interfaces

#ifndef _CMDWRITER_H_
#define _CMDWRITER_H_

#include <vector>
#include <stdint.h>
#include <string.h>

namespace pm4_profile {

// User defined options for flusing cache
typedef struct FlushCacheOptions_ {
  bool l1, l2;
  bool icache, kcache;
  bool l1_vol, l2_vol, kcache_vol;
  FlushCacheOptions_() {
    l1 = l2 = icache = kcache = false;
    l1_vol = l2_vol = kcache_vol = false;
  };
} FlushCacheOptions;

/// @brief Interface to build a list of Gpu commands into a byte
/// buffer. Classes implementing this interface are used to translate
/// various Gpu commands as byte stream.
///
/// @note: The Api does not require implementations to be thread safe.
/// Users are therefore required to be access in a serialized manner.
class CmdBuf {
 public:
  /// Default destructor.
  virtual ~CmdBuf() {}

  /// @brief Resets the command buffer object. All of the commands
  /// previously packed into the buffer are lost i.e. the number of
  /// bytes in command stream is reset.
  ///
  /// @note: This convenience Api is provided to allow reuse of the
  /// command buffer object.
  ///
  /// @return bool true if successful, false otherwise.
  virtual bool Reset(void) = 0;

  /// @brief Appends input command into a buffer that could
  /// be queried for its size and other properties. The append
  /// does not verify the contents.
  ///
  /// @param cmd Buffer containing one or more instances of Gpu commands
  ///
  /// @param size size of the Gpu commands in bytes.
  ///
  /// @return void
  virtual void AppendCommand(const void* cmd, uint32_t size) = 0;

  /// @brief Returns the total size (in bytes) of the accumulated commands.
  ///
  /// @return size_t size of Gpu commands in bytes
  virtual size_t Size() const = 0;

 private:
  /// Indexes the command buffer by dwords.  Allows accessing constants
  /// in an assembled command buffer.
  virtual uint32_t& operator[](size_t index) = 0;

  friend class CommandWriter;
};

/// @brief Implements the interface CmdBuf and thus can be used to
/// translate various Gpu commands as byte stream.
///
/// @note: The Api does not require implementations to be thread safe.
/// Users are therefore required to be access in a serialized manner.
class DefaultCmdBuf : public CmdBuf {
 public:
  /// @brief Append the command into the underlying buffer
  ///
  /// @param cmd Buffer containing one or more instances of Gpu commands
  ///
  /// @param size Size of Gpu command(s) in bytes
  ///
  /// @retur void
  virtual void AppendCommand(const void* cmd, uint32_t size) {
    memcpy(ReserveCmdbufSpace(size), cmd, size);
  }

  /// @brief Resets the Gpu command buffer
  bool Reset() {
    cmdbuf_.clear();
    return true;
  }

  /// Size of Gpu commands in bytes in the underlying buffer
  size_t Size() const { return cmdbuf_.size() * sizeof(StorageType); }

  /// Address of the start of accumulated commands.
  const void* Base() const { return &cmdbuf_[0]; }

 private:
  /// @brief Returns reference to the value of Gpu command buffer
  /// at specified index
  ///
  /// @param index Specifies the buffer index whose value is needed
  ///
  /// @return uint32_t & Reference of the value being returned
  uint32_t& operator[](size_t index) { return cmdbuf_[index]; }

  /// @brief Increase Gpu command buffer by specified size
  ///
  /// @param size Size in bytes by which command buffer should
  /// be resized.
  ///
  /// @return void * Pointer into the buffer where the next
  /// command can be written
  void* ReserveCmdbufSpace(std::size_t size) {
    const size_t len = cmdbuf_.size();
    cmdbuf_.resize(len + size / sizeof(StorageType));
    return &cmdbuf_[len];
  }

  /// @brief Defines Gpu command buffer as a vector of StorageType
  typedef uint32_t StorageType;
  std::vector<StorageType> cmdbuf_;
};

/// @brief Specifies the public interface of CommandWriter for use by
/// clients to build Gpu command streams.
class CommandWriter {
 public:
  /// @brief These enums specify the operation to perform in the packet
  /// generated by BuildAtomicPacket. The commenting for each enum uses
  /// the arguments to the function BuildAtomicPacket to express the
  /// resulting operation.
  enum AtomicType {

    /// *destination = *destination + 1;
    kAtomicTypeIncrement,

    /// *destination = *destination - 1;
    kAtomicTypeDecrement,

    /// if (*destination == compare) *destination = value;
    kAtomicTypeCompareAndSwap,

    /// while (*destination != compare);
    /// *destination = value;
    kAtomicTypeBlockingCompareAndSwap,

    /// *destination = *destination + value;
    kAtomicAdd,

    /// *destination = *destination - value;
    kAtomicSubtract,

    /// *destination = value;
    kAtomicSwap
  };

  /// @brief These enums specify the VGT EVENT TYPE to issue and wait for.
  /// Command Processor (CP) uses these events to communicate with SPI to
  /// learn about outstanding waves and determine kernel completion.
  enum VgtEventType {

    /// Enable Performance Counters
    kPerfCntrsStart,

    /// Disable Performance Counters
    kPerfCntrsStop,

    /// Read Performance Counters
    kPerfCntrsSample,

    /// Enable a Thread Trace session
    kThrdTraceStart,

    /// Disable a Thread Trace session
    kThrdTraceStop,

    /// Enable flushing of thread trace buffers
    kThrdTraceFlush,

    /// Enables resetting of BASE register to its last value
    /// including flushing of thread trace buffers. This could
    /// be used to toggle between two buffers so as to allow
    /// collection of large token data
    kThrdTraceFinish
  };

  /// @brief Returns the Dword that encodes a No-Op for the CP
  ///
  /// @return uint32_t Dword that can be used to populate a Pm4
  /// command queue.
  ///
  virtual uint32_t GetNoOpCmd() = 0;

  /// @brief Build an instance of Barrier command and copy it into
  /// the input commmand buffer
  ///
  /// @param cmdbuf Pointer to command buffer which is updated with
  /// an instance of Barrier command.
  ///
  /// @return void
  virtual void BuildBarrierCommand(CmdBuf* cmdbuf) = 0;

  /// @brief Builds the Gpu command to reference indirectly a stream
  /// of other Gpu commands. The launch command is then copied into
  /// the command buffer parameter.
  ///
  /// @param cmdBuf command buffer to be appended with launch command
  ///
  /// @param cmd_addr Address of command buffer carrying command stream
  ///
  /// @param cmd_size Size of dispatch command stream in bytes
  ///
  /// @return void
  virtual void BuildIndirectBufferCmd(CmdBuf* cmdbuf, const void* cmd_addr,
                                      std::size_t cmd_size) = 0;

  /// @brief Build a Gpu command that triggers an event whose type
  /// is specified by input parameter. It then copies it into the input
  /// command buffer
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param event Id of Event to be triggered by Gpu
  ///
  /// @return void
  virtual void BuildWriteEventPacket(CmdBuf* cmdbuf, uint32_t event) = 0;

  /// @bried Builds a Gpu command to wait until condition is realized
  ///
  /// @param cmdbuf command buffer to be appended with launch command
  ///
  /// @param mem_space if the address is in memory or is a register offset
  ///
  /// @param wait_addr address to wait on
  ///
  /// @param func_eq true means equal, false means not-equal
  ///
  /// @param mask_val Mask to apply on value from addr in comparison
  ///
  /// @param wait_val value to apply for the func given above
  virtual void BuildWaitRegMemCommand(CmdBuf* cmdbuf, bool mem_space, uint64_t wait_addr,
                                      bool func_eq, uint32_t mask_val, uint32_t wait_val) = 0;

  virtual void BuildUpdateHostAddress(CmdBuf* cmdbuf, uint64_t* addr, int64_t value) = 0;

  /// @brief Build CP command to program a Gpu register
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  /// @param addr Register to be programmed
  /// @param value Value to write into register
  ///
  /// @return void
  virtual void BuildWriteUConfigRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) = 0;

  /// @brief Build and copy WriteShReg command
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param addr Offset of the register
  ///
  /// @param value Value to write into register
  ///
  /// @return void
  virtual void BuildWriteShRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) = 0;

  /// @brief Builds a Gpu command to flush Gpu caches and write a
  /// user defined value at a configurable location that is Gpu
  /// accessible.
  ///
  /// @param cmdBuf Command buffer to be appended with bottom of pipe
  /// notification command
  ///
  /// @param write_addr Address into which Gpu should write
  ///
  /// @param write_val Value to write into user provided address
  ///
  /// @param interrupt True if Gpu should raise an interrupt upon writing
  /// the user value
  ///
  /// @return void
  virtual void BuildBOPNotifyCmd(CmdBuf* cmdbuf, const void* write_addr, uint32_t write_val,
                                 bool intrpt) = 0;


  /// @brief Build a Gpu command that copies data from a specified
  /// source to destination
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param reg_to_mem flag to indicate if values are being read from a
  /// Register or a memory location
  ///
  /// @param src_addr_lo Low 32-bit Source address of the data to read from
  ///
  /// @param src_addr_hi High 32-bit Source address of the data to read from
  ///
  /// @param dst_addr Destination address for the data to be written to
  ///
  /// @param size Size of the data to be written
  ///
  /// @param  wait True if Gpu command should confirm the write operation
  /// operation has completed successfully
  ///
  /// @return void
  ///
  /// @NOTE Change interface to use void* for Src and void* for Dest
  virtual void BuildCopyDataPacket(CmdBuf* cmdbuf, uint32_t src_sel, uint32_t src_addr_lo,
                                   uint32_t src_addr_hi, uint32_t* dst_addr, uint32_t size,
                                   bool wait) = 0;

  /// @brief Build and copy a WaitIdle Gpu command into command buffer
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @return void
  virtual void BuildWriteWaitIdlePacket(CmdBuf* cmdbuf) = 0;

  // Will issue a VGT event including a cache flush later on
  virtual void BuildVgtEventPacket(CmdBuf* cmdbuf, uint32_t vgtEvent) = 0;

  /// @brief Build and copy a WriteRegister Gpu command into command buffer
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param addr Register into which to write
  ///
  /// @param value Value to write into register
  ///
  /// @return void
  virtual void BuildWriteRegisterPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) = 0;

  /// @brief Build and copy a Gpu command to query the status of a
  /// WriteEvent into command buffer
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param event Id of Event whose status is to be queried
  ///
  /// @param addr Address to update the status of WriteEvent operation
  ///
  /// @return void
  virtual void BuildWriteEventQueryPacket(CmdBuf* cmdBuf, uint32_t event, uint32_t* addr) = 0;

  /// @brief Builds and copies a Gpu comamnd to peform user specified
  /// operation atomically. The various atomic operations on integers
  /// that are supported include: increment, decrement, add, subtract,
  /// compare-and-swap and swap. The operation to perform is specified
  /// by the enum AtomicType.
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param atomic_op Id of the atomic operation to perform
  ///
  /// @param addr Pointer to the memory block where atomic operation
  /// would be performed
  ///
  /// @param value New value to write if atomic operation can be performed
  ///
  /// @param compare Value to compare if atomic operation is a compare-and-swap
  ///
  /// @return void
  virtual void BuildAtomicPacket(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint32_t* addr,
                                 uint32_t value = 0, uint32_t compare = 0) = 0;

  /// @brief Builds and copies a Gpu comamnd to peform user specified
  /// operation atomically. The various atomic operations on integers
  /// that are supported include: increment, decrement, add, subtract,
  /// compare-and-swap and swap. The operation to perform is specified
  /// by the enum AtomicType.
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param atomic_op Id of the atomic operation to perform
  ///
  /// @param addr Pointer to the memory block where atomic operation
  /// would be performed
  ///
  /// @param value New value to write if atomic operation can be performed
  ///
  /// @param compare Value to compare if atomic operation is a compare-and-swap
  ///
  /// @return void
  virtual void BuildAtomicPacket64(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint64_t* addr,
                                   uint64_t value = 0, uint64_t compare = 0) = 0;

  /// @brief Returns the size of an atomic packet
  ///
  /// @return size_t Size of atomic packet
  virtual size_t SizeOfAtomicPacket() const = 0;

  /// @brief Build and copy a Gpu command that will tell command processor
  /// to conditionally execute or skip the next sequence of packets.
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param signal Pointer to an integer that tells the command processor
  /// whether to skip or execute the next block of packets. If it is set
  /// to 0 the following packets will be skipped, else it will execute the
  /// following packets
  ///
  /// @param count The number of dwords in the following packet stream
  /// that will be conditionally executed
  ///
  /// @return void
  virtual void BuildConditionalExecute(CmdBuf* cmdbuf, uint32_t* signal, uint16_t count) = 0;

  /// @brief Builds a CP command to write user specified value
  /// at a user specified address. The command is then copied
  /// into the command buffer for submission to a device queue.
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param write_addr Address into which CP will write the user
  /// specified value
  ///
  /// @param write_value Value to write into the user specified address
  ///
  /// @return void
  virtual void BuildWriteDataCommand(CmdBuf* cmdbuf, uint32_t* write_addr,
                                     uint32_t write_value) = 0;

  /// @brief Builds a CP command to write user specified value
  /// at a user specified address. The command is then copied
  /// into the command buffer for submission to a device queue.
  ///
  /// @param cmdbuf Pointer to command buffer to be appended
  ///
  /// @param write_addr Address into which CP will write the user
  /// specified value
  ///
  /// @param write_value Value to write into the user specified address
  ///
  /// @return void
  virtual void BuildWriteData64Command(CmdBuf* cmdbuf, uint64_t* write_addr,
                                       uint64_t write_value) = 0;

  /// Writes into input buffer Gpu commands to flush its cache. It is
  /// necessary that the buffer provided for flush commands is large
  /// enough to accommodate the full set of commands. It should be at
  /// least 512 bytes.
  ///
  /// @param tsCmdBuf Buffer to write commands to.
  /// @param writeAddr Registered address into which GPU should write
  /// a user provided value upon executing the flush commands.
  /// @param writeVal User provided value written by GPU at user provided
  /// address, upon executing the flush commands.
  ///
  /// @return void
  virtual void BuildFlushCacheCmd(CmdBuf* cmdbuf, FlushCacheOptions* options, uint32_t* writeAddr,
                                  uint32_t writeVal) = 0;

  /// @brief Builds Gpu command to copy data from source to destination
  /// buffer using DMA engine.
  ///
  /// @param cmdbuf Buffer updated with Gpu copy command
  /// @param srcAddr Address of source buffer address
  /// @param dstAddr Address of destination buffer address
  /// @param copySize Size of data to copy in bytes
  /// @param waitForCompletion if command should wait for copying to complete
  virtual void BuildDmaDataPacket(CmdBuf* cmdbuf, uint32_t* srcAddrLo, uint32_t* dstAddr,
                                  uint32_t copySize, bool waitForCompletion) = 0;

  /// @brief Release resources used by CommandWriter
  virtual ~CommandWriter(){};

 protected:
  /// @brief Return the reference to a value in the command buffer
  uint32_t& IndexBuffer(CmdBuf* cmdbuf, uint32_t index) { return (*cmdbuf)[index]; }
};

/// @brief Returns the lower 32-bits of a value
inline uint32_t Low32(uint64_t u) { return (u & 0xFFFFFFFFUL); }

/// @brief Returns the upper 32-bits of a value
inline uint32_t High32(uint64_t u) { return (u >> 32); }

/// @brief Returns the lower 32-bits of an address
inline uint32_t PtrLow32(const void* p) {
  return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
}

/// @brief Returns the upper 32-bits of an address
inline uint32_t PtrHigh32(const void* p) {
  uint32_t hi_32 = 0;
#ifdef HSA_LARGE_MODEL
  hi_32 = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p) >> 32);
  static_assert(sizeof(void*) == 8, "HSA_LARGE_MODEL is not set properly here!");
#else
  static_assert(sizeof(void*) == 4, "HSA_LARGE_MODEL is not set properly here!");
#endif
  return hi_32;
}

}  // pm4_profile

#endif  // _CMDWRITER_H_