hsa-runtime integration

Change-Id: I48968966ffe164218ebff88d0e3a1268e96bf1dd


[ROCm/ROCR-Runtime commit: 4174f07fd1]

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/CMakeLists.txt

@@ -0,0 +1,10 @@
+#
+# Source files for Rocr Cmdwriter
+#
+set ( CmdWriterSrcs gfx8_cmdwriter.cpp )
+set ( CmdWriterSrcs ${CmdWriterSrcs} gfx9_cmdwriter.cpp )
+
+#
+# Build Cmdwriter as a Static Library object
+#
+add_library ( ${CMDWRITER_LIB} STATIC ${CmdWriterSrcs} )

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/cmdwriter.h

@@ -0,0 +1,498 @@
+// cmdwriter.h
+// Header file for CommandWriter and CmdBuf interfaces
+
+#ifndef _CMDWRITER_H_
+#define _CMDWRITER_H_
+
+#include <vector>
+#include <stdint.h>
+#include <string.h>
+
+namespace pm4_profile {
+
+// User defined options for flusing cache
+typedef struct FlushCacheOptions_ {
+  bool l1, l2;
+  bool icache, kcache;
+  bool l1_vol, l2_vol, kcache_vol;
+  FlushCacheOptions_() {
+    l1 = l2 = icache = kcache = false;
+    l1_vol = l2_vol = kcache_vol = false;
+  };
+} FlushCacheOptions;
+
+/// @brief Interface to build a list of Gpu commands into a byte
+/// buffer. Classes implementing this interface are used to translate
+/// various Gpu commands as byte stream.
+///
+/// @note: The Api does not require implementations to be thread safe.
+/// Users are therefore required to be access in a serialized manner.
+class CmdBuf {
+ public:
+  /// Default destructor.
+  virtual ~CmdBuf() {}
+
+  /// @brief Resets the command buffer object. All of the commands
+  /// previously packed into the buffer are lost i.e. the number of
+  /// bytes in command stream is reset.
+  ///
+  /// @note: This convenience Api is provided to allow reuse of the
+  /// command buffer object.
+  ///
+  /// @return bool true if successful, false otherwise.
+  virtual bool Reset(void) = 0;
+
+  /// @brief Appends input command into a buffer that could
+  /// be queried for its size and other properties. The append
+  /// does not verify the contents.
+  ///
+  /// @param cmd Buffer containing one or more instances of Gpu commands
+  ///
+  /// @param size size of the Gpu commands in bytes.
+  ///
+  /// @return void
+  virtual void AppendCommand(const void* cmd, uint32_t size) = 0;
+
+  /// @brief Returns the total size (in bytes) of the accumulated commands.
+  ///
+  /// @return size_t size of Gpu commands in bytes
+  virtual size_t Size() const = 0;
+
+ private:
+  /// Indexes the command buffer by dwords.  Allows accessing constants
+  /// in an assembled command buffer.
+  virtual uint32_t& operator[](size_t index) = 0;
+
+  friend class CommandWriter;
+};
+
+/// @brief Implements the interface CmdBuf and thus can be used to
+/// translate various Gpu commands as byte stream.
+///
+/// @note: The Api does not require implementations to be thread safe.
+/// Users are therefore required to be access in a serialized manner.
+class DefaultCmdBuf : public CmdBuf {
+ public:
+  /// @brief Append the command into the underlying buffer
+  ///
+  /// @param cmd Buffer containing one or more instances of Gpu commands
+  ///
+  /// @param size Size of Gpu command(s) in bytes
+  ///
+  /// @retur void
+  virtual void AppendCommand(const void* cmd, uint32_t size) {
+    memcpy(ReserveCmdbufSpace(size), cmd, size);
+  }
+
+  /// @brief Resets the Gpu command buffer
+  bool Reset() {
+    cmdbuf_.clear();
+    return true;
+  }
+
+  /// Size of Gpu commands in bytes in the underlying buffer
+  size_t Size() const { return cmdbuf_.size() * sizeof(StorageType); }
+
+  /// Address of the start of accumulated commands.
+  const void* Base() const { return &cmdbuf_[0]; }
+
+ private:
+  /// @brief Returns reference to the value of Gpu command buffer
+  /// at specified index
+  ///
+  /// @param index Specifies the buffer index whose value is needed
+  ///
+  /// @return uint32_t & Reference of the value being returned
+  uint32_t& operator[](size_t index) { return cmdbuf_[index]; }
+
+  /// @brief Increase Gpu command buffer by specified size
+  ///
+  /// @param size Size in bytes by which command buffer should
+  /// be resized.
+  ///
+  /// @return void * Pointer into the buffer where the next
+  /// command can be written
+  void* ReserveCmdbufSpace(std::size_t size) {
+    const size_t len = cmdbuf_.size();
+    cmdbuf_.resize(len + size / sizeof(StorageType));
+    return &cmdbuf_[len];
+  }
+
+  /// @brief Defines Gpu command buffer as a vector of StorageType
+  typedef uint32_t StorageType;
+  std::vector<StorageType> cmdbuf_;
+};
+
+/// @brief Specifies the public interface of CommandWriter for use by
+/// clients to build Gpu command streams.
+class CommandWriter {
+ public:
+  /// @brief These enums specify the operation to perform in the packet
+  /// generated by BuildAtomicPacket. The commenting for each enum uses
+  /// the arguments to the function BuildAtomicPacket to express the
+  /// resulting operation.
+  enum AtomicType {
+
+    /// *destination = *destination + 1;
+    kAtomicTypeIncrement,
+
+    /// *destination = *destination - 1;
+    kAtomicTypeDecrement,
+
+    /// if (*destination == compare) *destination = value;
+    kAtomicTypeCompareAndSwap,
+
+    /// while (*destination != compare);
+    /// *destination = value;
+    kAtomicTypeBlockingCompareAndSwap,
+
+    /// *destination = *destination + value;
+    kAtomicAdd,
+
+    /// *destination = *destination - value;
+    kAtomicSubtract,
+
+    /// *destination = value;
+    kAtomicSwap
+  };
+
+  /// @brief These enums specify the VGT EVENT TYPE to issue and wait for.
+  /// Command Processor (CP) uses these events to communicate with SPI to
+  /// learn about outstanding waves and determine kernel completion.
+  enum VgtEventType {
+
+    /// Enable Performance Counters
+    kPerfCntrsStart,
+
+    /// Disable Performance Counters
+    kPerfCntrsStop,
+
+    /// Read Performance Counters
+    kPerfCntrsSample,
+
+    /// Enable a Thread Trace session
+    kThrdTraceStart,
+
+    /// Disable a Thread Trace session
+    kThrdTraceStop,
+
+    /// Enable flushing of thread trace buffers
+    kThrdTraceFlush,
+
+    /// Enables resetting of BASE register to its last value
+    /// including flushing of thread trace buffers. This could
+    /// be used to toggle between two buffers so as to allow
+    /// collection of large token data
+    kThrdTraceFinish
+  };
+
+  /// @brief Returns the Dword that encodes a No-Op for the CP
+  ///
+  /// @return uint32_t Dword that can be used to populate a Pm4
+  /// command queue.
+  ///
+  virtual uint32_t GetNoOpCmd() = 0;
+
+  /// @brief Build an instance of Barrier command and copy it into
+  /// the input commmand buffer
+  ///
+  /// @param cmdbuf Pointer to command buffer which is updated with
+  /// an instance of Barrier command.
+  ///
+  /// @return void
+  virtual void BuildBarrierCommand(CmdBuf* cmdbuf) = 0;
+
+  /// @brief Builds the Gpu command to reference indirectly a stream
+  /// of other Gpu commands. The launch command is then copied into
+  /// the command buffer parameter.
+  ///
+  /// @param cmdBuf command buffer to be appended with launch command
+  ///
+  /// @param cmd_addr Address of command buffer carrying command stream
+  ///
+  /// @param cmd_size Size of dispatch command stream in bytes
+  ///
+  /// @return void
+  virtual void BuildIndirectBufferCmd(CmdBuf* cmdbuf, const void* cmd_addr,
+                                      std::size_t cmd_size) = 0;
+
+  /// @brief Build a Gpu command that triggers an event whose type
+  /// is specified by input parameter. It then copies it into the input
+  /// command buffer
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param event Id of Event to be triggered by Gpu
+  ///
+  /// @return void
+  virtual void BuildWriteEventPacket(CmdBuf* cmdbuf, uint32_t event) = 0;
+
+  /// @bried Builds a Gpu command to wait until condition is realized
+  ///
+  /// @param cmdbuf command buffer to be appended with launch command
+  ///
+  /// @param mem_space if the address is in memory or is a register offset
+  ///
+  /// @param wait_addr address to wait on
+  ///
+  /// @param func_eq true means equal, false means not-equal
+  ///
+  /// @param mask_val Mask to apply on value from addr in comparison
+  ///
+  /// @param wait_val value to apply for the func given above
+  virtual void BuildWaitRegMemCommand(CmdBuf* cmdbuf, bool mem_space, uint64_t wait_addr,
+                                      bool func_eq, uint32_t mask_val, uint32_t wait_val) = 0;
+
+  virtual void BuildUpdateHostAddress(CmdBuf* cmdbuf, uint64_t* addr, int64_t value) = 0;
+
+  /// @brief Build CP command to program a Gpu register
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  /// @param addr Register to be programmed
+  /// @param value Value to write into register
+  ///
+  /// @return void
+  virtual void BuildWriteUConfigRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) = 0;
+
+  /// @brief Build and copy WriteShReg command
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param addr Offset of the register
+  ///
+  /// @param value Value to write into register
+  ///
+  /// @return void
+  virtual void BuildWriteShRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) = 0;
+
+  /// @brief Builds a Gpu command to flush Gpu caches and write a
+  /// user defined value at a configurable location that is Gpu
+  /// accessible.
+  ///
+  /// @param cmdBuf Command buffer to be appended with bottom of pipe
+  /// notification command
+  ///
+  /// @param write_addr Address into which Gpu should write
+  ///
+  /// @param write_val Value to write into user provided address
+  ///
+  /// @param interrupt True if Gpu should raise an interrupt upon writing
+  /// the user value
+  ///
+  /// @return void
+  virtual void BuildBOPNotifyCmd(CmdBuf* cmdbuf, const void* write_addr, uint32_t write_val,
+                                 bool intrpt) = 0;
+
+
+  /// @brief Build a Gpu command that copies data from a specified
+  /// source to destination
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param reg_to_mem flag to indicate if values are being read from a
+  /// Register or a memory location
+  ///
+  /// @param src_addr_lo Low 32-bit Source address of the data to read from
+  ///
+  /// @param src_addr_hi High 32-bit Source address of the data to read from
+  ///
+  /// @param dst_addr Destination address for the data to be written to
+  ///
+  /// @param size Size of the data to be written
+  ///
+  /// @param  wait True if Gpu command should confirm the write operation
+  /// operation has completed successfully
+  ///
+  /// @return void
+  ///
+  /// @NOTE Change interface to use void* for Src and void* for Dest
+  virtual void BuildCopyDataPacket(CmdBuf* cmdbuf, uint32_t src_sel, uint32_t src_addr_lo,
+                                   uint32_t src_addr_hi, uint32_t* dst_addr, uint32_t size,
+                                   bool wait) = 0;
+
+  /// @brief Build and copy a WaitIdle Gpu command into command buffer
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @return void
+  virtual void BuildWriteWaitIdlePacket(CmdBuf* cmdbuf) = 0;
+
+  // Will issue a VGT event including a cache flush later on
+  virtual void BuildVgtEventPacket(CmdBuf* cmdbuf, uint32_t vgtEvent) = 0;
+
+  /// @brief Build and copy a WriteRegister Gpu command into command buffer
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param addr Register into which to write
+  ///
+  /// @param value Value to write into register
+  ///
+  /// @return void
+  virtual void BuildWriteRegisterPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) = 0;
+
+  /// @brief Build and copy a Gpu command to query the status of a
+  /// WriteEvent into command buffer
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param event Id of Event whose status is to be queried
+  ///
+  /// @param addr Address to update the status of WriteEvent operation
+  ///
+  /// @return void
+  virtual void BuildWriteEventQueryPacket(CmdBuf* cmdBuf, uint32_t event, uint32_t* addr) = 0;
+
+  /// @brief Builds and copies a Gpu comamnd to peform user specified
+  /// operation atomically. The various atomic operations on integers
+  /// that are supported include: increment, decrement, add, subtract,
+  /// compare-and-swap and swap. The operation to perform is specified
+  /// by the enum AtomicType.
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param atomic_op Id of the atomic operation to perform
+  ///
+  /// @param addr Pointer to the memory block where atomic operation
+  /// would be performed
+  ///
+  /// @param value New value to write if atomic operation can be performed
+  ///
+  /// @param compare Value to compare if atomic operation is a compare-and-swap
+  ///
+  /// @return void
+  virtual void BuildAtomicPacket(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint32_t* addr,
+                                 uint32_t value = 0, uint32_t compare = 0) = 0;
+
+  /// @brief Builds and copies a Gpu comamnd to peform user specified
+  /// operation atomically. The various atomic operations on integers
+  /// that are supported include: increment, decrement, add, subtract,
+  /// compare-and-swap and swap. The operation to perform is specified
+  /// by the enum AtomicType.
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param atomic_op Id of the atomic operation to perform
+  ///
+  /// @param addr Pointer to the memory block where atomic operation
+  /// would be performed
+  ///
+  /// @param value New value to write if atomic operation can be performed
+  ///
+  /// @param compare Value to compare if atomic operation is a compare-and-swap
+  ///
+  /// @return void
+  virtual void BuildAtomicPacket64(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint64_t* addr,
+                                   uint64_t value = 0, uint64_t compare = 0) = 0;
+
+  /// @brief Returns the size of an atomic packet
+  ///
+  /// @return size_t Size of atomic packet
+  virtual size_t SizeOfAtomicPacket() const = 0;
+
+  /// @brief Build and copy a Gpu command that will tell command processor
+  /// to conditionally execute or skip the next sequence of packets.
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param signal Pointer to an integer that tells the command processor
+  /// whether to skip or execute the next block of packets. If it is set
+  /// to 0 the following packets will be skipped, else it will execute the
+  /// following packets
+  ///
+  /// @param count The number of dwords in the following packet stream
+  /// that will be conditionally executed
+  ///
+  /// @return void
+  virtual void BuildConditionalExecute(CmdBuf* cmdbuf, uint32_t* signal, uint16_t count) = 0;
+
+  /// @brief Builds a CP command to write user specified value
+  /// at a user specified address. The command is then copied
+  /// into the command buffer for submission to a device queue.
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param write_addr Address into which CP will write the user
+  /// specified value
+  ///
+  /// @param write_value Value to write into the user specified address
+  ///
+  /// @return void
+  virtual void BuildWriteDataCommand(CmdBuf* cmdbuf, uint32_t* write_addr,
+                                     uint32_t write_value) = 0;
+
+  /// @brief Builds a CP command to write user specified value
+  /// at a user specified address. The command is then copied
+  /// into the command buffer for submission to a device queue.
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  ///
+  /// @param write_addr Address into which CP will write the user
+  /// specified value
+  ///
+  /// @param write_value Value to write into the user specified address
+  ///
+  /// @return void
+  virtual void BuildWriteData64Command(CmdBuf* cmdbuf, uint64_t* write_addr,
+                                       uint64_t write_value) = 0;
+
+  /// Writes into input buffer Gpu commands to flush its cache. It is
+  /// necessary that the buffer provided for flush commands is large
+  /// enough to accommodate the full set of commands. It should be at
+  /// least 512 bytes.
+  ///
+  /// @param tsCmdBuf Buffer to write commands to.
+  /// @param writeAddr Registered address into which GPU should write
+  /// a user provided value upon executing the flush commands.
+  /// @param writeVal User provided value written by GPU at user provided
+  /// address, upon executing the flush commands.
+  ///
+  /// @return void
+  virtual void BuildFlushCacheCmd(CmdBuf* cmdbuf, FlushCacheOptions* options, uint32_t* writeAddr,
+                                  uint32_t writeVal) = 0;
+
+  /// @brief Builds Gpu command to copy data from source to destination
+  /// buffer using DMA engine.
+  ///
+  /// @param cmdbuf Buffer updated with Gpu copy command
+  /// @param srcAddr Address of source buffer address
+  /// @param dstAddr Address of destination buffer address
+  /// @param copySize Size of data to copy in bytes
+  /// @param waitForCompletion if command should wait for copying to complete
+  virtual void BuildDmaDataPacket(CmdBuf* cmdbuf, uint32_t* srcAddrLo, uint32_t* dstAddr,
+                                  uint32_t copySize, bool waitForCompletion) = 0;
+
+  /// @brief Release resources used by CommandWriter
+  virtual ~CommandWriter(){};
+
+ protected:
+  /// @brief Return the reference to a value in the command buffer
+  uint32_t& IndexBuffer(CmdBuf* cmdbuf, uint32_t index) { return (*cmdbuf)[index]; }
+};
+
+/// @brief Returns the lower 32-bits of a value
+inline uint32_t Low32(uint64_t u) { return (u & 0xFFFFFFFFUL); }
+
+/// @brief Returns the upper 32-bits of a value
+inline uint32_t High32(uint64_t u) { return (u >> 32); }
+
+/// @brief Returns the lower 32-bits of an address
+inline uint32_t PtrLow32(const void* p) {
+  return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
+}
+
+/// @brief Returns the upper 32-bits of an address
+inline uint32_t PtrHigh32(const void* p) {
+  uint32_t hi_32 = 0;
+#ifdef HSA_LARGE_MODEL
+  hi_32 = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p) >> 32);
+  static_assert(sizeof(void*) == 8, "HSA_LARGE_MODEL is not set properly here!");
+#else
+  static_assert(sizeof(void*) == 4, "HSA_LARGE_MODEL is not set properly here!");
+#endif
+  return hi_32;
+}
+
+}  // pm4_profile
+
+#endif  // _CMDWRITER_H_

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/gfx8_cmds.h

@@ -0,0 +1,161 @@
+#ifndef _GFX8_CMDS_H_
+#define _GFX8_CMDS_H_
+
+#include "gfxip/gfx8/si_ci_vi_merged_enum.h"
+#include "gfxip/gfx8/si_ci_vi_merged_mask.h"
+#include "gfxip/gfx8/si_ci_vi_merged_offset.h"
+#include "gfxip/gfx8/si_ci_vi_merged_registers.h"
+#include "gfxip/gfx8/si_ci_vi_merged_typedef.h"
+#include "gfxip/gfx8/si_ci_vi_merged_pm4_it_opcodes.h"
+#include "gfxip/gfx8/si_pm4defs.h"
+
+namespace pm4_profile {
+
+namespace gfx8 {
+
+// Desc: Defines the Gpu command to dispatch a kernel. It embeds
+// various Gpu hardware specific data structures for initialization
+// and configuration before a dispatch begins to run
+struct DispatchTemplate {
+  // Desc: Structure used to initialize the group dimensions
+  // of a kernel dispatch and if performance counters are enabled
+  struct DispatchDimensionRegs {
+    PM4CMDSETDATA cmd_set_data;
+    regCOMPUTE_START_X compute_start_x;
+    regCOMPUTE_START_Y compute_start_y;
+    regCOMPUTE_START_Z compute_start_z;
+    regCOMPUTE_NUM_THREAD_X compute_num_thread_x;
+    regCOMPUTE_NUM_THREAD_Y compute_num_thread_y;
+    regCOMPUTE_NUM_THREAD_Z compute_num_thread_z;
+    regCOMPUTE_PIPELINESTAT_ENABLE__CI__VI compute_pipelinestat_enable;
+  } dimension_regs;
+
+  // Desc: Structure used to initialize kernel Isa, trap
+  // handler, trap handler buffer, number of SGPR and VGPR
+  // registers needed, amount of Group memory and LDS needed,
+  // Rounding mode for Floating point numbers, etc.
+  struct DispatchProgramRegs {
+    PM4CMDSETDATA cmd_set_data;
+    regCOMPUTE_PGM_LO compute_pgm_lo;
+    regCOMPUTE_PGM_HI compute_pgm_hi;
+    regCOMPUTE_TBA_LO compute_tba_lo;
+    regCOMPUTE_TBA_HI compute_tba_hi;
+    regCOMPUTE_TMA_LO compute_tma_lo;
+    regCOMPUTE_TMA_HI compute_tma_hi;
+    regCOMPUTE_PGM_RSRC1 compute_pgm_rsrc1;
+    regCOMPUTE_PGM_RSRC2 compute_pgm_rsrc2;
+  } program_regs;
+
+  // Desc: Structure used to initialize parameters related to
+  // thread management i.e. number of waves to issue and number
+  // of Compute Units to use
+  struct DispatchResourceRegs {
+    PM4CMDSETDATA cmd_set_data;
+    regCOMPUTE_RESOURCE_LIMITS compute_resource_limits;
+    regCOMPUTE_STATIC_THREAD_MGMT_SE0 compute_static_thread_mgmt_se0;
+    regCOMPUTE_STATIC_THREAD_MGMT_SE1 compute_static_thread_mgmt_se1;
+    regCOMPUTE_TMPRING_SIZE compute_tmpring_size;
+    regCOMPUTE_STATIC_THREAD_MGMT_SE2__CI__VI compute_static_thread_mgmt_se2;
+    regCOMPUTE_STATIC_THREAD_MGMT_SE3__CI__VI compute_static_thread_mgmt_se3;
+    regCOMPUTE_RESTART_X__CI__VI compute_restart_x;
+    regCOMPUTE_RESTART_Y__CI__VI compute_restart_y;
+    regCOMPUTE_RESTART_Z__CI__VI compute_restart_z;
+    regCOMPUTE_THREAD_TRACE_ENABLE__CI__VI compute_thread_trace_enable;
+  } resource_regs;
+
+  // Desc: Structure used to pass handles of the Aql dispatch
+  // packet, Aql queue, Kernel argument address block, Scratch
+  // buffer
+  struct DispatchComputeUserDataRegs {
+    PM4CMDSETDATA cmd_set_data;
+    uint32_t compute_user_data[16];
+  } compute_user_data_regs;
+
+  // Desc: Structure used to configure Cache flush policy
+  // and dimensions of total work size
+  PM4CMDDISPATCHDIRECT dispatch_direct;
+};
+
+// Desc: Structure used to issue a Gpu Barrier command
+struct BarrierTemplate {
+  PM4CMDEVENTWRITE event_write;
+};
+
+// Desc: Structure used to configure the flushing
+// of various caches - instruction, constants, L1
+// and L2
+struct AcquireMemTemplate {
+  PM4CMDACQUIREMEM acquire_mem;
+};
+
+// Desc: Structure used to reference another Gpu command
+// indirectly. Generally used to reference a list of Gpu
+// commands (dispatch cmds) indirectly
+struct LaunchTemplate {
+  PM4CMDINDIRECTBUFFER indirect_buffer;
+};
+
+// Desc: Structure used to determine the end of
+// a kernel including cache flushes and writing to
+// a user configurable memory location
+struct EndofKernelNotifyTemplate {
+  PM4CMDRELEASEMEM release_mem;
+};
+
+// Desc: Strucuture used to perform various atomic
+// operations - add, subtract, increment, etc
+struct AtomicTemplate {
+  PM4CMDATOMIC atomic;
+};
+
+// Desc: Structure used to conditionalize the execution
+// of a Gpu command stream
+struct ConditionalExecuteTemplate {
+  PM4CMDCONDEXEC_CI conditional;
+};
+
+// Desc: PM4 command to write a 32-bit value into a memory
+// location accessible to Gpu
+struct WriteDataTemplate {
+  PM4CMDWRITEDATA write_data;
+  uint32_t write_data_value;
+};
+
+// Desc: PM4 command to write a 64-bit value into a memory
+// location accessible to Gpu
+struct WriteData64Template {
+  PM4CMDWRITEDATA write_data;
+  uint64_t write_data_value;
+};
+
+// Desc: PM4 command to wait for a certain event before proceeding
+// to process another command on the queue
+struct WaitRegMemTemplate {
+  PM4CMDWAITREGMEM wait_reg_mem;
+};
+
+// Desc: Initializer for commands that set shader registers
+template <class T> void GenerateSetShRegHeader(T* pm4, uint32_t reg_addr) {
+  pm4->cmd_set_data.header.u32All =
+      PM4_TYPE_3_HDR(IT_SET_SH_REG, sizeof(T) / sizeof(uint32_t), ShaderCompute, 0);
+  pm4->cmd_set_data.regOffset = reg_addr - PERSISTENT_SPACE_START;
+}
+
+// Desc: Initializer for various Gpu command headers
+template <class T> void GenerateCmdHeader(T* pm4, IT_OpCodeType op_code) {
+  pm4->header.u32All = PM4_TYPE_3_HDR(op_code, sizeof(T) / sizeof(uint32_t), ShaderCompute, 0);
+}
+
+// Desc: Initializer for commands that set configuration registers
+template <class T> void GenerateSetConfigRegHeader(T* pm4, uint32_t reg_addr) {
+  pm4->cmd_set_data.header.u32All =
+      PM4_TYPE_3_HDR(IT_SET_CONFIG_REG, sizeof(T) / sizeof(uint32_t), ShaderCompute, 0);
+  pm4->cmd_set_data.regOffset = reg_addr - CONFIG_SPACE_START;
+}
+
+
+}  // gfx8
+
+}  // pm4_profile
+
+#endif  //  _GFX8_CMDS_H_

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/gfx8_cmdwriter.cpp

@@ -0,0 +1,765 @@
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+#include <string.h>
+#include <assert.h>
+
+#include "gfx8_cmdwriter.h"
+#include "gfxip/gfx8/gfx8_utils.h"
+
+// RELEASE MEM DST SEL Definitions
+#define RELEASE_MEM_DST_SEL_MEMORY_CONTROLLER 0
+#define RELEASE_MEM_DST_SEL_TC_L2 1
+
+// RELEASE MEM CACHE POLICY Definitions
+#define RELEASE_MEM_CACHE_POLICY_LRU 0
+#define RELEASE_MEM_CACHE_POLICY_STREAM 1
+#define RELEASE_MEM_CACHE_POLICY_BYPASS 2
+
+template <class T> static void PrintPm4Packet(const T& command, const char* name) {
+#if !defined(NDEBUG)
+  uint32_t* cmd = (uint32_t*)&command;
+  uint32_t size = sizeof(command) / sizeof(uint32_t);
+  std::ostringstream oss;
+  oss << "'" << name << "' size(" << std::dec << size << ")";
+  std::clog << std::setw(40) << std::left << oss.str() << ":";
+  for (uint32_t idx = 0; idx < size; idx++) {
+    std::clog << " " << std::hex << std::setw(8) << std::setfill('0') << cmd[idx];
+  }
+  std::clog << std::setfill(' ') << std::endl;
+#endif
+}
+
+#define APPEND_COMMAND_WRAPPER(cmdbuf, command)                                                    \
+  PrintPm4Packet(command, __FUNCTION__);                                                           \
+  AppendCommand(cmdbuf, command);
+
+namespace pm4_profile {
+namespace gfx8 {
+
+template <class T> void Gfx8CmdWriter::AppendCommand(CmdBuf* cmdbuf, const T& command) {
+  cmdbuf->AppendCommand(&command, sizeof(command));
+}
+
+void Gfx8CmdWriter::InitializeAtomicTemplate() {
+  memset(&atomic_template_.atomic, 0, sizeof(atomic_template_));
+  GenerateCmdHeader(&atomic_template_.atomic, IT_ATOMIC_MEM__CI);
+
+  if (atc_support_) {
+    const uint32_t kAtcShift = 24;
+    atomic_template_.atomic.ordinal2 |= 1 << kAtcShift;
+  }
+}
+
+void Gfx8CmdWriter::InitializeConditionalTemplate() {
+  memset(&conditional_template_.conditional, 0, sizeof(conditional_template_));
+  gfx8::GenerateCmdHeader(&conditional_template_.conditional, IT_COND_EXEC);
+
+  if (atc_support_) {
+    const uint32_t kAtcShift = 24;
+    conditional_template_.conditional.ordinal4 |= 1 << kAtcShift;
+  }
+}
+
+void Gfx8CmdWriter::InitializeLaunchTemplate() {
+  memset(&launch_template_, 0, sizeof(launch_template_));
+
+  GenerateCmdHeader(&launch_template_.indirect_buffer, IT_INDIRECT_BUFFER);
+  launch_template_.indirect_buffer.CI.valid = true;
+}
+
+void Gfx8CmdWriter::InitializeWriteDataTemplate() {
+  // Set the header of write data command
+  memset(&write_data_template_, 0, sizeof(write_data_template_));
+
+  // Initialize the header of command packet
+  PM4CMDWRITEDATA* command = &(write_data_template_.write_data);
+  uint32_t cmd_size = sizeof(write_data_template_) / sizeof(uint32_t);
+  command->ordinal1 = PM4_TYPE_3_HDR(IT_WRITE_DATA, cmd_size, ShaderCompute, 0);
+
+  // Set the ATC bit of command template - specifies if the address
+  // belongs to system memory
+  write_data_template_.write_data.atc__CI = (atc_support_) ? 1 : 0;
+
+  // Set the bit to confirm the write operation and cache policy
+  write_data_template_.write_data.wrConfirm = 1;
+  write_data_template_.write_data.cachePolicy__CI = WRITE_DATA_CACHE_POLICY_BYPASS;
+
+  // Specify the module that will execute the write data command
+  write_data_template_.write_data.engineSel = WRITE_DATA_ENGINE_ME;
+
+  // Specify the class to which the write destination belongs
+  write_data_template_.write_data.dstSel = WRITE_DATA_DST_SEL_MEMORY_ASYNC;
+}
+
+void Gfx8CmdWriter::InitializeWriteData64Template() {
+  // Set the header of write data command
+  memset(&write_data64_template_, 0, sizeof(write_data64_template_));
+
+  // Initialize the header of command packet
+  PM4CMDWRITEDATA* command = &(write_data64_template_.write_data);
+  uint32_t cmd_size = sizeof(write_data64_template_) / sizeof(uint32_t);
+  command->ordinal1 = PM4_TYPE_3_HDR(IT_WRITE_DATA, cmd_size, ShaderCompute, 0);
+
+  // Set the ATC bit of command template - specifies if the address
+  // belongs to system memory
+  write_data64_template_.write_data.atc__CI = (atc_support_) ? 1 : 0;
+
+  // Set the bit to confirm the write operation and cache policy
+  write_data64_template_.write_data.wrConfirm = 1;
+  write_data64_template_.write_data.cachePolicy__CI = WRITE_DATA_CACHE_POLICY_BYPASS;
+
+  // Specify the module that will execute the write data command
+  write_data64_template_.write_data.engineSel = WRITE_DATA_ENGINE_ME;
+
+  // Specify the class to which the write destination belongs
+  // write_data64_template_.write_data.dstSel = WRITE_DATA_DST_SEL_TCL2;
+  // TODO: For Hawaii bring up only.
+  write_data64_template_.write_data.dstSel = WRITE_DATA_DST_SEL_MEMORY_ASYNC;
+}
+
+void Gfx8CmdWriter::InitializeBarrierTemplate() {
+  memset(&pending_dispatch_template_, 0, sizeof(pending_dispatch_template_));
+
+  gfx8::GenerateCmdHeader(&pending_dispatch_template_.event_write, IT_EVENT_WRITE);
+  pending_dispatch_template_.event_write.eventType = CS_PARTIAL_FLUSH;
+  pending_dispatch_template_.event_write.eventIndex = EventTypeToIndexTable[CS_PARTIAL_FLUSH];
+}
+
+void Gfx8CmdWriter::InitializeAcquireMemTemplate() {
+  memset(&invalidate_cache_template_, 0, sizeof(invalidate_cache_template_));
+
+  gfx8::GenerateCmdHeader(&invalidate_cache_template_.acquire_mem, IT_ACQUIRE_MEM__CI__VI);
+  invalidate_cache_template_.acquire_mem.cpCoherBase.u32All = 0x00;
+  invalidate_cache_template_.acquire_mem.cpCoherBaseHi.u32All = 0x00;
+  invalidate_cache_template_.acquire_mem.cpCoherSize.u32All = 0xFFFFFFFF;
+  invalidate_cache_template_.acquire_mem.cpCoherSizeHi.u32All = 0xFF;
+  invalidate_cache_template_.acquire_mem.pollInterval = 0;
+}
+
+void Gfx8CmdWriter::InitializeWaitRegMemTemplate() {
+  memset(&wait_reg_mem_template_, 0, sizeof(wait_reg_mem_template_));
+
+  gfx8::GenerateCmdHeader(&wait_reg_mem_template_.wait_reg_mem, IT_WAIT_REG_MEM);
+  wait_reg_mem_template_.wait_reg_mem.atc__CI = (atc_support_) ? 1 : 0;
+  wait_reg_mem_template_.wait_reg_mem.cachePolicy__CI = 2;  // bypass
+  wait_reg_mem_template_.wait_reg_mem.pollInterval = 0;
+  wait_reg_mem_template_.wait_reg_mem.engine = WAIT_REG_MEM_ENGINE_ME;
+}
+
+Gfx8CmdWriter::Gfx8CmdWriter(bool atc_support, bool pcie_atomic_support) {
+  // Initialize various state variables related to
+  // atomic operations and atc support
+  pcie_atomic_support_ = pcie_atomic_support;
+  atc_support_ = atc_support;
+
+  InitializeLaunchTemplate();
+  InitializeAtomicTemplate();
+  InitializeConditionalTemplate();
+  InitializeWriteDataTemplate();
+  InitializeWriteData64Template();
+  InitializeBarrierTemplate();
+  InitializeAcquireMemTemplate();
+  InitializeWaitRegMemTemplate();
+}
+
+void Gfx8CmdWriter::BuildWaitRegMemCommand(CmdBuf* cmdbuf, bool mem_space, uint64_t wait_addr,
+                                           bool func_eq, uint32_t mask_val, uint32_t wait_val) {
+  gfx8::WaitRegMemTemplate wait_cmd = wait_reg_mem_template_;
+
+  // Apply the space to which addr belongs
+  if (mem_space) {
+    wait_cmd.wait_reg_mem.memSpace = WAIT_REG_MEM_SPACE_MEMORY;
+  } else {
+    wait_cmd.wait_reg_mem.memSpace = WAIT_REG_MEM_SPACE_REGISTER;
+  }
+
+  // Apply the function - equal / not equal desired by user
+  if (func_eq) {
+    wait_cmd.wait_reg_mem.function = WAIT_REG_MEM_FUNC_EQUAL;
+  } else {
+    wait_cmd.wait_reg_mem.function = WAIT_REG_MEM_FUNC_NOT_EQUAL;
+  }
+
+  // Apply the mask on value at address/register
+  wait_cmd.wait_reg_mem.mask = mask_val;
+
+  // Value to use in applying equal / not equal function
+  wait_cmd.wait_reg_mem.reference = wait_val;
+
+  // Update upper 32 bit address if addr is not a register
+  if (mem_space) {
+    assert(!(wait_addr & 0x3) && "WaitRegMem address must be 4 byte aligned");
+  }
+  wait_cmd.wait_reg_mem.pollAddressLo = Low32(wait_addr);
+  if (mem_space) {
+    wait_cmd.wait_reg_mem.pollAddressHi = High32(wait_addr);
+  }
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, wait_cmd);
+}
+
+void Gfx8CmdWriter::BuildUpdateHostAddress(CmdBuf* cmdbuf, uint64_t* addr, int64_t value) {
+  // If Atomics are supported, use it
+  if (pcie_atomic_support_) {
+    BuildAtomicPacket64(cmdbuf, CommandWriter::AtomicType::kAtomicSwap, (volatile uint64_t*)addr,
+                        value);
+    return;
+  }
+
+  BuildWriteData64Command(cmdbuf, addr, value);
+  return;
+}
+
+void Gfx8CmdWriter::BuildIndirectBufferCmd(CmdBuf* cmdbuf, const void* cmd_addr,
+                                           std::size_t cmd_size) {
+  gfx8::LaunchTemplate launch = launch_template_;
+
+  launch.indirect_buffer.ibBaseLo = PtrLow32(cmd_addr);
+  launch.indirect_buffer.ibBaseHi = PtrHigh32(cmd_addr);
+  launch.indirect_buffer.CI.ibSize = cmd_size / sizeof(uint32_t);
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, launch);
+}
+
+void Gfx8CmdWriter::BuildBOPNotifyCmd(CmdBuf* cmdbuf, const void* write_addr, uint32_t write_val,
+                                      bool interrupt) {
+  // Initialize the command including its header
+  gfx8::EndofKernelNotifyTemplate eopCmd;
+  memset(&eopCmd, 0, sizeof(eopCmd));
+  gfx8::GenerateCmdHeader(&eopCmd.release_mem, IT_RELEASE_MEM__CI__VI);
+
+  // Program CP to wait until following event is notified by SPI
+  eopCmd.release_mem.eventType = BOTTOM_OF_PIPE_TS;
+  eopCmd.release_mem.eventIndex = EventTypeToIndexTable[BOTTOM_OF_PIPE_TS];
+
+  // Program CP to perform various cache operations
+  // which complete before Write operation commences
+  eopCmd.release_mem.atc = atc_support_;
+  eopCmd.release_mem.l2Invlidate = true;
+  eopCmd.release_mem.l2WriteBack = true;
+
+  // Set destination as Memory with Write bypassing Cache
+  eopCmd.release_mem.cachePolicy = RELEASE_MEM_CACHE_POLICY_BYPASS;
+  eopCmd.release_mem.dstSel = RELEASE_MEM_DST_SEL_MEMORY_CONTROLLER;
+
+  // Program CP to write user specified value to user specified address
+  eopCmd.release_mem.ordinal4 = Low32(uint64_t(write_addr));
+  eopCmd.release_mem.addrHi = High32(uint64_t(write_addr));
+  eopCmd.release_mem.dataLo = Low32(write_val);
+  eopCmd.release_mem.dataHi = High32(write_val);
+  eopCmd.release_mem.dataSel = EVENTWRITEEOP_DATA_SEL_SEND_DATA32;
+
+  // Determine if host will poll or wait for interrupt
+  eopCmd.release_mem.intSel =
+      (interrupt == false) ? EVENTWRITEEOP_INT_SEL_NONE : EVENTWRITEEOP_INT_SEL_SEND_INT_ON_CONFIRM;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, eopCmd);
+}
+
+
+void Gfx8CmdWriter::BuildBarrierFenceCommands(CmdBuf* cmdbuf) {
+  gfx8::AcquireMemTemplate invalidate_src_caches = invalidate_cache_template_;
+
+  // wbINVL2 by default writes-back and invalidates both L1 and L2
+  invalidate_src_caches.acquire_mem.coherCntl =
+      CP_COHER_CNTL__TC_ACTION_ENA_MASK | CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK__CI__VI;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, invalidate_src_caches);
+}
+
+// PM4 packet for profilers
+#define PM4_PACKET3 (0xC0000000)
+#define PM4_PACKET3_CMD_SHIFT 8
+#define PM4_PACKET3_COUNT_SHIFT 16
+
+#define PACKET3(cmd, count)                                                                        \
+  (PM4_PACKET3 | (((count)-1) << PM4_PACKET3_COUNT_SHIFT) | ((cmd) << PM4_PACKET3_CMD_SHIFT))
+
+// Structure to store the event PM4 packet
+typedef struct WriteRegPacket_ { uint32_t item[3]; } WriteRegPacket;
+
+typedef struct WriteEventPacket_ { uint32_t item[7]; } WriteEventPacket;
+
+void Gfx8CmdWriter::BuildWriteEventPacket(CmdBuf* cmdbuf, uint32_t event) {
+  PM4CMDEVENTWRITE cp_event_initiator;
+  cp_event_initiator.ordinal1 = PACKET3(IT_EVENT_WRITE, 1);
+  cp_event_initiator.ordinal2 = 0;
+
+  VGT_EVENT_TYPE eventType = Reserved_0x00;
+  switch (event) {
+    case kPerfCntrsStart:
+      eventType = PERFCOUNTER_START;
+      break;
+    case kPerfCntrsStop:
+      eventType = PERFCOUNTER_STOP;
+      break;
+    case kPerfCntrsSample:
+      eventType = PERFCOUNTER_SAMPLE;
+      break;
+    default:
+      assert(false && "Illegal VGT Event Id");
+  }
+
+  cp_event_initiator.eventType = eventType;
+  cp_event_initiator.eventIndex = EventTypeToIndexTable[eventType];
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, cp_event_initiator);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildWriteUnshadowRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] = (PM4_TYPE_3_HDR(IT_SET_UCONFIG_REG__CI__VI, 1 + PM4_CMD_SET_CONFIG_REG_DWORDS,
+                                   ShaderGraphics, 0));
+  packet.item[1] = (addr - UCONFIG_SPACE_START__CI__VI);
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildWriteUConfigRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] = (PM4_TYPE_3_HDR(IT_SET_UCONFIG_REG__CI__VI, 1 + PM4_CMD_SET_CONFIG_REG_DWORDS,
+                                   ShaderCompute, 0));
+  packet.item[1] = (addr - UCONFIG_SPACE_START__CI__VI);
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildWriteShRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] = (PM4_TYPE_3_HDR(IT_SET_SH_REG, 1 + PM4_CMD_SET_SH_REG_DWORDS, ShaderCompute, 0));
+  packet.item[1] = (addr - PERSISTENT_SPACE_START);
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildCopyDataPacket(CmdBuf* cmdbuf, uint32_t src_sel, uint32_t src_addr_lo,
+                                        uint32_t src_addr_hi, uint32_t* dst_addr, uint32_t size,
+                                        bool wait) {
+  PM4CMDCOPYDATA cmd_data;
+  memset(&cmd_data, 0, sizeof(PM4CMDCOPYDATA));
+
+  cmd_data.header.u32All = PACKET3(IT_COPY_DATA, 5);
+
+  cmd_data.srcAtc__CI = atc_support_;
+  cmd_data.srcCachePolicy__CI = COPY_DATA_SRC_CACHE_POLICY_BYPASS;
+  cmd_data.srcSel = src_sel;
+
+  cmd_data.dstAtc__CI = atc_support_;
+  cmd_data.dstSel = COPY_DATA_SEL_DST_ASYNC_MEMORY;
+  cmd_data.dstCachePolicy__CI = COPY_DATA_DST_CACHE_POLICY_BYPASS;
+
+  uint32_t dst_addr_lo, dst_addr_hi;
+
+  dst_addr_lo = PtrLow32(dst_addr);
+  dst_addr_hi = PtrHigh32(dst_addr);
+
+  cmd_data.srcAddressLo = src_addr_lo;
+  cmd_data.srcAddressHi = src_addr_hi;
+  cmd_data.dstAddressLo = dst_addr_lo;
+  cmd_data.dstAddressHi = dst_addr_hi;
+
+  cmd_data.countSel = size;
+  cmd_data.wrConfirm = wait;
+  cmd_data.engineSel = COPY_DATA_ENGINE_ME;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, cmd_data);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildCacheFlushPacket(CmdBuf* cmdbuf) {
+  WriteEventPacket packet;
+  packet.item[0] = PACKET3(IT_ACQUIRE_MEM__CI__VI, 6);
+  packet.item[1] = 0x28C00000;
+  packet.item[2] = 0xFFFFFFFF;
+  packet.item[3] = 0;
+  packet.item[4] = 0;
+  packet.item[5] = 0;
+  packet.item[6] = 0x00000004;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+}
+
+void Gfx8CmdWriter::BuildWriteWaitIdlePacket(CmdBuf* cmdbuf) {
+  BuildBarrierCommand(cmdbuf);
+  BuildCacheFlushPacket(cmdbuf);
+  return;
+}
+
+// Will issue a VGT event including a cache flush later on
+void Gfx8CmdWriter::BuildVgtEventPacket(CmdBuf* cmdbuf, uint32_t vgtEvent) {
+  PM4CMDEVENTWRITE cp_event_initiator;
+
+  cp_event_initiator.ordinal1 = PACKET3(IT_EVENT_WRITE, 1);
+  cp_event_initiator.ordinal2 = 0;
+
+  VGT_EVENT_TYPE eventType = Reserved_0x00;
+  switch (vgtEvent) {
+    case kPerfCntrsStart:
+      eventType = PERFCOUNTER_START;
+      break;
+    case kPerfCntrsStop:
+      eventType = PERFCOUNTER_STOP;
+      break;
+    case kPerfCntrsSample:
+      eventType = PERFCOUNTER_SAMPLE;
+      break;
+    case kThrdTraceStart:
+      eventType = THREAD_TRACE_START;
+      break;
+    case kThrdTraceStop:
+      eventType = THREAD_TRACE_STOP;
+      break;
+    case kThrdTraceFlush:
+      eventType = THREAD_TRACE_FLUSH;
+      break;
+    case kThrdTraceFinish:
+      eventType = THREAD_TRACE_FINISH;
+      break;
+    default:
+      assert(false && "Illegal VGT Event Id");
+  }
+
+  cp_event_initiator.eventType = eventType;
+  cp_event_initiator.eventIndex = EventTypeToIndexTable[eventType];
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, cp_event_initiator);
+
+  // Check If I should be issuing a cache flush operation as well
+  // test and remove it
+  BuildCacheFlushPacket(cmdbuf);
+  return;
+}
+
+void Gfx8CmdWriter::BuildWriteRegisterPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] =
+      (PM4_TYPE_3_HDR(IT_SET_CONFIG_REG, 1 + PM4_CMD_SET_CONFIG_REG_DWORDS, ShaderGraphics, 0));
+  packet.item[1] = addr - CONFIG_SPACE_START;
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildWriteEventQueryPacket(CmdBuf* cmdbuf, uint32_t event, uint32_t* addr) {
+  PM4CMDEVENTWRITEQUERY cp_event_initiator;
+  cp_event_initiator.ordinal1 = PACKET3(IT_EVENT_WRITE, 3);
+  cp_event_initiator.ordinal2 = 0;
+
+  // Update switch statements you want to support
+  VGT_EVENT_TYPE eventType = Reserved_0x00;
+  switch (event) {
+    default:
+      assert(false && "Illegal VGT Event Id");
+  }
+
+  cp_event_initiator.eventType = eventType;
+  cp_event_initiator.eventIndex = EventTypeToIndexTable[eventType];
+
+  // set the address
+  uint32_t addrLo = PtrLow32(addr);
+  uint32_t addrHi = PtrHigh32(addr);
+  ((addrLo & 0x7) != 0) ? assert(false) : assert(true);
+
+  cp_event_initiator.ordinal3 = 0;
+  cp_event_initiator.ordinal4 = 0;
+  cp_event_initiator.addressLo = addrLo;
+  cp_event_initiator.addressHi = addrHi;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, cp_event_initiator);
+
+  return;
+}
+
+void Gfx8CmdWriter::BuildBarrierCommand(CmdBuf* cmdBuf) {
+  APPEND_COMMAND_WRAPPER(cmdBuf, pending_dispatch_template_);
+}
+
+void Gfx8CmdWriter::WriteUserData(uint32_t* dst_addr, uint32_t count, const void* src_addr) {
+  memcpy(dst_addr, src_addr, count * sizeof(uint32_t));
+}
+
+
+void Gfx8CmdWriter::BuildAtomicPacket(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint32_t* addr,
+                                      uint32_t value, uint32_t compare) {
+  gfx8::AtomicTemplate atomic = atomic_template_;
+
+  // make sure the destination adddress is aligned
+  uint32_t address_low = PtrLow32((void*)addr);
+  uint32_t address_high = PtrHigh32((void*)addr);
+  assert(!(address_low & 0x7) && "destination address must be 8 byte aligned");
+
+  atomic.atomic.addressLo = address_low;
+  atomic.atomic.addressHi = address_high;
+
+  switch (atomic_op) {
+    case CommandWriter::kAtomicTypeIncrement: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_ADD_RTN_32;
+      atomic.atomic.srcDataLo = 1;
+      break;
+    }
+    case CommandWriter::kAtomicTypeDecrement: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_SUB_RTN_32;
+      atomic.atomic.srcDataLo = 1;
+      break;
+    }
+    case CommandWriter::kAtomicTypeCompareAndSwap: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_CMPSWAP_RTN_32;
+      atomic.atomic.srcDataLo = value;
+      atomic.atomic.cmpDataLo = compare;
+      break;
+    }
+    case CommandWriter::kAtomicTypeBlockingCompareAndSwap: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_CMPSWAP_RTN_32;
+      atomic.atomic.srcDataLo = value;
+      atomic.atomic.cmpDataLo = compare;
+      atomic.atomic.command = 1;
+      atomic.atomic.loopInterval = 128;
+      break;
+    }
+    case CommandWriter::kAtomicAdd: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_ADD_RTN_32;
+      atomic.atomic.srcDataLo = value;
+      break;
+    }
+    case CommandWriter::kAtomicSubtract: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_SUB_RTN_32;
+      atomic.atomic.srcDataLo = value;
+      break;
+    }
+    case CommandWriter::kAtomicSwap: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_SWAP_RTN_32;
+      atomic.atomic.srcDataLo = value;
+      break;
+    }
+  }
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, atomic);
+}
+
+void Gfx8CmdWriter::BuildAtomicPacket64(CmdBuf* cmdbuf, AtomicType atomic_op,
+                                        volatile uint64_t* addr, uint64_t value, uint64_t compare) {
+  AtomicTemplate atomic = atomic_template_;
+
+  // make sure the destination adddress is aligned
+  uint32_t address_low = PtrLow32((void*)addr);
+  uint32_t address_high = PtrHigh32((void*)addr);
+  assert(!(address_low & 0x7) && "destination address must be 8 byte aligned");
+
+  atomic.atomic.addressLo = address_low;
+  atomic.atomic.addressHi = address_high;
+
+  atomic.atomic.atc = (atc_support_) ? 1 : 0;
+  atomic.atomic.cachePolicy = 2;
+
+  switch (atomic_op) {
+    case CommandWriter::kAtomicTypeIncrement: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_ADD_RTN_64;
+      atomic.atomic.srcDataLo = 1;
+      break;
+    }
+    case CommandWriter::kAtomicTypeDecrement: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_SUB_RTN_64;
+      atomic.atomic.srcDataLo = 1;
+      break;
+    }
+    case CommandWriter::kAtomicTypeCompareAndSwap: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_CMPSWAP_RTN_64;
+      atomic.atomic.srcDataLo = Low32(value);
+      atomic.atomic.srcDataHi = High32(value);
+      atomic.atomic.cmpDataLo = Low32(compare);
+      atomic.atomic.cmpDataHi = High32(compare);
+      break;
+    }
+    case CommandWriter::kAtomicTypeBlockingCompareAndSwap: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_CMPSWAP_RTN_64;
+      atomic.atomic.srcDataLo = Low32(value);
+      atomic.atomic.srcDataHi = High32(value);
+      atomic.atomic.cmpDataLo = Low32(compare);
+      atomic.atomic.cmpDataHi = High32(compare);
+      atomic.atomic.command = 1;
+      atomic.atomic.loopInterval = 128;
+      break;
+    }
+    case CommandWriter::kAtomicAdd: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_ADD_RTN_64;
+      atomic.atomic.srcDataLo = Low32(value);
+      atomic.atomic.srcDataHi = High32(value);
+      break;
+    }
+    case CommandWriter::kAtomicSubtract: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_SUB_RTN_64;
+      atomic.atomic.srcDataLo = Low32(value);
+      atomic.atomic.srcDataHi = High32(value);
+      break;
+    }
+    case CommandWriter::kAtomicSwap: {
+      atomic.atomic.atomOp = TC_OP_ATOMIC_SWAP_RTN_64;
+      atomic.atomic.srcDataLo = Low32(value);
+      atomic.atomic.srcDataHi = High32(value);
+      break;
+    }
+  }
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, atomic);
+}
+
+size_t Gfx8CmdWriter::SizeOfAtomicPacket() const {
+  return sizeof(AtomicTemplate) / sizeof(uint32_t);
+}
+
+void Gfx8CmdWriter::BuildConditionalExecute(CmdBuf* cmdbuf, uint32_t* signal, uint16_t count) {
+  ConditionalExecuteTemplate conditional = conditional_template_;
+
+  uint32_t address_low = PtrLow32(signal);
+  uint32_t address_high = PtrHigh32(signal);
+  assert(!(address_low & 0x7) && "destination address must be 8 byte aligned");
+
+  conditional.conditional.boolAddrLo = address_low;
+  conditional.conditional.boolAddrHi = address_high;
+  conditional.conditional.execCount = count;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, conditional);
+}
+
+void Gfx8CmdWriter::BuildWriteDataCommand(CmdBuf* cmdbuf, uint32_t* write_addr,
+                                          uint32_t write_value) {
+  // Copy the initialize command packet
+  gfx8::WriteDataTemplate command = write_data_template_;
+
+  // Encode the user specified value to write
+  command.write_data_value = write_value;
+
+  // Encode the user specified address to write to
+  command.write_data.dstAddrLo = PtrLow32(write_addr);
+  command.write_data.dstAddrHi = PtrHigh32(write_addr);
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, command);
+}
+
+void Gfx8CmdWriter::BuildWriteData64Command(CmdBuf* cmdbuf, uint64_t* write_addr,
+                                            uint64_t write_value) {
+  // Copy the initialize command packet
+  gfx8::WriteData64Template command = write_data64_template_;
+
+  // Encode the user specified value to write
+  command.write_data_value = write_value;
+
+  // Encode the user specified address to write to
+  command.write_data.dstAddrLo = PtrLow32(write_addr);
+  command.write_data.dstAddrHi = PtrHigh32(write_addr);
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, command);
+}
+
+void Gfx8CmdWriter::BuildFlushCacheCmd(CmdBuf* cmdbuf, FlushCacheOptions* options,
+                                       uint32_t* writeAddr, uint32_t writeVal) {
+  PM4CMDACQUIREMEM flushCmd;
+  memset(&flushCmd, 0, sizeof(flushCmd));
+
+  // Verify write back address is valid. Note that this address is NOT
+  // used on CI. But to have a same interface as that on SI, we keep
+  // the address argument in this function. Thus, this check always pass
+  // no matter the address is NULL or not.
+  (writeAddr == NULL) ? assert(true) : assert(true);
+
+  // Initialize the command header
+  gfx8::GenerateCmdHeader(&flushCmd, IT_ACQUIRE_MEM__CI__VI);
+
+  // Specify the base address of memory being synchronized.
+  // The starting address is indicated as follows: bits [0-48].
+  flushCmd.cpCoherBase.u32All = 0;
+  flushCmd.cpCoherBaseHi.u32All = 0;
+
+  // Specify the size of memory being synchronized. It is indicated
+  // as follows:
+  //    COHER_SIZE_256B_MASK = 0xffffffffL
+  //    COHER_SIZE_HI_256B_MASK__CI__VI = 0x000000ffL
+  flushCmd.cpCoherSize.u32All = CP_COHER_SIZE__COHER_SIZE_256B_MASK;
+  flushCmd.cpCoherSizeHi.u32All = CP_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK__CI__VI;
+
+  // Periodicity of polling - interval to wait from the time
+  // of unsuccessful polling result is returned and a new
+  // poll is issued
+  flushCmd.pollInterval = 0x04;
+
+  // Program Coherence Control Register. Initialize L2 Cache flush
+  // for Non-Coherent memory blocks
+  uint32_t coher_cntl = 0;
+
+  coher_cntl |= (options->l1) ? CP_COHER_CNTL__TCL1_ACTION_ENA_MASK : 0;
+  coher_cntl |= (options->l2)
+      ? (CP_COHER_CNTL__TC_ACTION_ENA_MASK | CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK__CI__VI)
+      : 0;
+  coher_cntl |= (options->icache) ? CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK : 0;
+  coher_cntl |= (options->kcache) ? CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK : 0;
+  flushCmd.coherCntl = coher_cntl;
+
+  // Copy AcquireMem command buffer stream
+  APPEND_COMMAND_WRAPPER(cmdbuf, flushCmd);
+  return;
+}
+
+void Gfx8CmdWriter::BuildDmaDataPacket(CmdBuf* cmdbuf, uint32_t* srcAddr, uint32_t* dstAddr,
+                                       uint32_t copySize, bool waitForConfirm) {
+  PM4CMDDMADATA cmdDmaData;
+  memset(&cmdDmaData, 0, sizeof(PM4CMDDMADATA));
+  cmdDmaData.header.u32All =
+      (PM4_TYPE_3_HDR(IT_DMA_DATA__CI__VI, PM4_CMD_DMA_DATA_DWORDS, ShaderCompute, 0));
+
+  // Id of Micro Engine
+  cmdDmaData.engine = 0;
+
+  // Specify attributes of source buffer such as its
+  // location, ATC property, Cache policy and Volatile
+  // A value of 1 for cache policy means to Stream
+  cmdDmaData.srcSel = 0;
+  cmdDmaData.srcATC = atc_support_;
+  cmdDmaData.srcCachePolicy = 1;
+  cmdDmaData.srcVolatile = 0;
+
+  // Specify attributes of destination buffer such as
+  // its location, ATC property, Cache policy and Volatile
+  // A value of 1 for cache policy means to Stream
+  cmdDmaData.dstSel = 0;
+  cmdDmaData.dstATC = atc_support_;
+  cmdDmaData.dstCachePolicy = 1;
+  cmdDmaData.dstVolatile = 0;
+
+  // Specify the source and destination addr
+  cmdDmaData.srcAddrHi = PtrHigh32(srcAddr);
+  cmdDmaData.srcAddrLoOrData = PtrLow32(srcAddr);
+  cmdDmaData.dstAddrLo = PtrLow32(dstAddr);
+  cmdDmaData.dstAddrHi = PtrHigh32(dstAddr);
+
+  // Number of bytes to copy. The command restricts
+  // the size to be (2 MB - 1) - 21 Bits
+  assert(copySize < 0x1FFFFF);
+  cmdDmaData.command.byteCount = copySize;
+
+  // Indicate that DMA Cmd should wait if its source
+  // is the destination of a previous DMA Cmd
+  cmdDmaData.command.rawWait = waitForConfirm;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, cmdDmaData);
+  return;
+}
+
+}  // gfx8
+}  // pm4_profile

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/gfx8_cmdwriter.h

@@ -0,0 +1,201 @@
+#ifndef _GFX8_CMDWRITER_H_
+#define _GFX8_CMDWRITER_H_
+
+#include "cmdwriter.h"
+#include "gfx8_cmds.h"
+
+namespace pm4_profile {
+
+namespace gfx8 {
+
+/// @brief class Gfx8CmdWriter implements the virtual class CommandWriter
+/// for Sea Islands (CI) and VI chipset
+class Gfx8CmdWriter : public CommandWriter {
+ public:
+  Gfx8CmdWriter(bool atc_support, bool pcie_atomic_support);
+
+  /// @brief Dword specifying NOOP command for SI/CI/VI chipsets. The macro
+  /// populates the NOOP command which is 32-bits wide. The second parameter,
+  /// the COUNT field of NOOP command, specifies the number of Dwords to skip.
+  /// To skip ZERO Dwords the value should be set to 0x3FFF. Since the macro
+  /// decrements the second parameter by TWO, an artifact of its definition,
+  /// the value is incremented by TWO to 0x4001 (0x3FFF + 2).
+  ///
+  inline uint32_t GetNoOpCmd() {
+    static const uint32_t nopCmd = PM4_TYPE_3_HDR(IT_NOP, 0x4001, ShaderCompute, 0);
+    return nopCmd;
+  }
+
+  void BuildBarrierCommand(CmdBuf* cmdBuf);
+
+  void BuildIndirectBufferCmd(CmdBuf* cmdbuf, const void* cmd_addr, std::size_t cmd_size);
+
+  void BuildBOPNotifyCmd(CmdBuf* cmdbuf, const void* write_addr, uint32_t write_val,
+                         bool interrupt);
+
+  void BuildBarrierFenceCommands(CmdBuf* cmdbuf);
+
+  void BuildWriteEventPacket(CmdBuf* cmdbuf, uint32_t event);
+
+  void BuildWaitRegMemCommand(CmdBuf* cmdbuf, bool mem_space, uint64_t wait_addr, bool func_eq,
+                              uint32_t mask_val, uint32_t wait_val);
+
+  void BuildWriteUnshadowRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  /// @brief Build CP command to program a Gpu register
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  /// @param addr Register to be programmed
+  /// @param value Value to write into register
+  ///
+  /// @return void
+  void BuildWriteUConfigRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  void BuildWriteShRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  void BuildCopyDataPacket(CmdBuf* cmdbuf, uint32_t src_sel, uint32_t src_addr_lo,
+                           uint32_t src_addr_hi, uint32_t* dst_addr, uint32_t size, bool wait);
+
+  void BuildWriteWaitIdlePacket(CmdBuf* cmdbuf);
+
+  // Will issue a VGT event including a cache flush later on
+  void BuildVgtEventPacket(CmdBuf* cmdbuf, uint32_t vgtEvent);
+
+  void BuildWriteRegisterPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  void BuildWriteEventQueryPacket(CmdBuf* cmdbuf, uint32_t event, uint32_t* addr);
+
+  void BuildAtomicPacket(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint32_t* addr,
+                         uint32_t value, uint32_t compare);
+
+  void BuildAtomicPacket64(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint64_t* addr,
+                           uint64_t value = 0, uint64_t compare = 0);
+
+  size_t SizeOfAtomicPacket() const;
+
+  void BuildConditionalExecute(CmdBuf* cmdbuf, uint32_t* signal, uint16_t count);
+
+  void BuildWriteDataCommand(CmdBuf* cmdbuf, uint32_t* write_addr, uint32_t write_value);
+
+  void BuildWriteData64Command(CmdBuf* cmdbuf, uint64_t* write_addr, uint64_t write_value);
+
+  void BuildCacheFlushPacket(CmdBuf* cmdbuf);
+
+  /// Writes into input buffer Gpu commands to flush its cache. It is
+  /// necessary that the buffer provided for flush commands is large
+  /// enough to accommodate the full set of commands. It should be at
+  /// least 512 bytes.
+  ///
+  /// @param tsCmdBuf Buffer to write commands to.
+  /// @param writeAddr Registered address into which GPU should write
+  /// a user provided value upon executing the flush commands.
+  /// @param writeVal User provided value written by GPU at user provided
+  /// address, upon executing the flush commands.
+  ///
+  /// @return void
+  void BuildFlushCacheCmd(CmdBuf* cmdBuf, FlushCacheOptions* options, uint32_t* writeAddr,
+                          uint32_t writeVal);
+
+  /// Builds Gpu command to copy data from source to destination buffer
+  /// using DMA engine.
+  ///
+  /// @param cmdbuf Buffer updated with Gpu copy command
+  /// @param srcAddr Address of source buffer address
+  /// @param dstAddr Address of destination buffer address
+  /// @param copySize Size of data to copy in bytes
+  /// @param waitForCompletion if command should wait for copying to complete
+  void BuildDmaDataPacket(CmdBuf* cmdBuf, uint32_t* srcAddr, uint32_t* dstAddr, uint32_t copySize,
+                          bool waitForCompletion);
+
+ protected:
+  /// @brief Copies data from source buffer to destination buffer
+  ///
+  /// @param dst_addr Address of destination buffer data
+  ///
+  /// @count Size of data to copy in 32-bit words
+  ///
+  /// @param src_addr Address of buffer containing source data
+  ///
+  /// @return void
+  virtual void WriteUserData(uint32_t* dst_addr, uint32_t count, const void* src_addr);
+
+  /// @brief Append an instance of Gpu command into input command buffer stream.
+  ///
+  /// @param cmdbuf CommandWriter object appended with anohter Gpu command
+  ///
+  /// @param cmd Gpu command to be appended into command buffer
+  ///
+  /// @return void
+  template <class T> void AppendCommand(CmdBuf* cmdbuf, const T& cmd);
+
+ private:
+  /// @brief Initializes a Gpu command which can be used to
+  /// reference a Gpu command stream indirectly
+  void InitializeLaunchTemplate();
+
+  /// @brief Initializes a Gpu command to perform atomic operations
+  ////
+  void InitializeAtomicTemplate();
+
+  /// @brief Initializes a Gpu command to allow conditional execution
+  /// of a Gpu command stream
+  void InitializeConditionalTemplate();
+
+  /// @brief Initializes a Gpu command to let command processor
+  /// wait for some update before letting other commands to be
+  /// processed
+  void InitializeWaitRegMemTemplate();
+
+  /// @brief Initializes the template for Barrier command.
+  /// Applications can use Barrier command to ensure their
+  /// command is executed only after all other commands have
+  /// completed their execution.
+  void InitializeBarrierTemplate();
+
+  void BuildUpdateHostAddress(CmdBuf* cmdbuf, uint64_t* addr, int64_t value);
+
+  /// @brief Initializes Acquire Memory command template. Users
+  /// can submit this command to invalidate Gpu caches - L1 and
+  /// or L2.
+  void InitializeAcquireMemTemplate();
+
+  /// @brief Initializes an instance of Write Data command
+  /// for use by an application
+  void InitializeWriteDataTemplate();
+  void InitializeWriteData64Template();
+
+  /// @brief Instance of Gpu command to reference dispatch commands
+  LaunchTemplate launch_template_;
+
+  /// @brief Instance of Gpu command to use in performing atomic operations
+  AtomicTemplate atomic_template_;
+
+  /// @brief Instance of Gpu command to use in conditional execution
+  /// of a command stream
+  ConditionalExecuteTemplate conditional_template_;
+
+  /// @brief Instance of Pm4 command WRITE_DATA
+  WriteDataTemplate write_data_template_;
+  WriteData64Template write_data64_template_;
+
+  /// @brief Instance of Pm4 command EVENT_WRITE
+  BarrierTemplate pending_dispatch_template_;
+
+  /// @brief Instance of Pm4 command ACQUIRE_MEM
+  AcquireMemTemplate invalidate_cache_template_;
+
+  /// @brief Instance of Pm4 command WAIT_REG_MEM
+  WaitRegMemTemplate wait_reg_mem_template_;
+
+  /// @brief ATC support.
+  bool atc_support_;
+
+  /// @brief PCIe atomic support.
+  bool pcie_atomic_support_;
+};
+
+}  // gfx8
+
+}  // pm4_profile
+
+#endif  //  _GFX8_CMDWRITER_H_

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/gfx9_cmds.h

@@ -0,0 +1,90 @@
+#ifndef _GFX9_CMDS_H_
+#define _GFX9_CMDS_H_
+
+#include "gfxip/gfx9/gfx9_utils.h"
+#include "gfxip/gfx9/gfx9_enum.h"
+#include "gfxip/gfx9/gfx9_mask.h"
+#include "gfxip/gfx9/gfx9_offset.h"
+#include "gfxip/gfx9/gfx9_typedef.h"
+#include "gfxip/gfx9/gfx9_registers.h"
+#include "gfxip/gfx9/gfx9_pm4_it_opcodes.h"
+#include "gfxip/gfx9/f32_mec_pm4_packets_vg10.h"
+#include "gfxip/gfx9/f32_pfp_pm4_packets_vg10.h"
+
+namespace pm4_profile {
+
+namespace gfx9 {
+
+/// @brief Initializer for commands that set shader registers
+template <class T> void GenerateSetShRegHeader(T* pm4, uint32_t reg_addr) {
+  pm4->cmd_set_data.header.u32All = PM4_TYPE3_HDR(IT_SET_SH_REG, sizeof(T) / sizeof(uint32_t));
+  pm4->cmd_set_data.bitfields2.reg_offset = reg_addr - PERSISTENT_SPACE_START;
+}
+
+// @brief Initializer for various Gpu command headers
+template <class T> void GenerateCmdHeader(T* pm4, IT_OpCodeType op_code) {
+  pm4->header.u32All = PM4_TYPE3_HDR(op_code, sizeof(T) / sizeof(uint32_t));
+}
+
+// @brief Initializer for commands that set configuration registers
+template <class T> void GenerateSetConfigRegHeader(T* pm4, uint32_t reg_addr) {
+  pm4->cmd_set_data.header.u32All = PM4_TYPE3_HDR(IT_SET_CONFIG_REG, sizeof(T) / sizeof(uint32_t));
+  pm4->cmd_set_data.bitfields2.reg_offset = reg_addr - CONFIG_SPACE_START;
+}
+
+/// @brief Structure used to issue a Gpu Barrier command
+struct BarrierTemplate {
+  PM4MEC_EVENT_WRITE event_write;
+};
+
+/// @brief Structure used to configure the flushing of
+/// various caches - instruction, constants, L1 and L2
+struct AcquireMemTemplate {
+  PM4MEC_ACQUIRE_MEM acquire_mem;
+};
+
+/// @brief Structure used to reference another Gpu command
+/// indirectly. Generally used to reference a list of Gpu
+/// commands (dispatch cmds) indirectly
+struct LaunchTemplate {
+  PM4MEC_INDIRECT_BUFFER indirect_buffer;
+};
+
+/// @brief Structure used to determine the end of
+/// a kernel including cache flushes and writing to
+/// a user configurable memory location
+struct EndofKernelNotifyTemplate {
+  PM4MEC_RELEASE_MEM release_mem;
+};
+
+// Desc: Strucuture used to perform various atomic
+// operations - add, subtract, increment, etc
+struct AtomicTemplate {
+  PM4MEC_ATOMIC_MEM atomic;
+};
+
+/// @brief PM4 command to write a 32-bit value into a memory
+/// location accessible to Gpu
+struct WriteDataTemplate {
+  PM4MEC_WRITE_DATA write_data;
+  uint32_t write_data_value;
+};
+
+/// @brief PM4 command to write a 64-bit value into a memory
+/// location accessible to Gpu
+struct WriteData64Template {
+  PM4MEC_WRITE_DATA write_data;
+  uint64_t write_data_value;
+};
+
+/// @brief PM4 command to wait for a certain event before proceeding
+/// to process another command on the queue
+struct WaitRegMemTemplate {
+  PM4MEC_WAIT_REG_MEM wait_reg_mem;
+};
+
+}  // gfx9
+
+}  // pm4_profile
+
+#endif  //  _GFX9_CMDS_H_

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/gfx9_cmdwriter.cpp

@@ -0,0 +1,743 @@
+#include <iostream>
+#include <iomanip>
+#include <sstream>
+#include <string.h>
+#include <assert.h>
+
+#include "gfx9_cmdwriter.h"
+
+template <class T> static void PrintPm4Packet(const T& command, const char* name) {
+#if !defined(NDEBUG)
+  uint32_t* cmd = (uint32_t*)&command;
+  uint32_t size = sizeof(command) / sizeof(uint32_t);
+  std::ostringstream oss;
+  oss << "'" << name << "' size(" << std::dec << size << ")";
+  std::clog << std::setw(40) << std::left << oss.str() << ":";
+  for (uint32_t idx = 0; idx < size; idx++) {
+    std::clog << " " << std::hex << std::setw(8) << std::setfill('0') << cmd[idx];
+  }
+  std::clog << std::setfill(' ') << std::endl;
+#endif
+}
+
+#define APPEND_COMMAND_WRAPPER(cmdbuf, command)                                                    \
+  PrintPm4Packet(command, __FUNCTION__);                                                           \
+  AppendCommand(cmdbuf, command);
+
+namespace pm4_profile {
+namespace gfx9 {
+
+template <class T> void Gfx9CmdWriter::AppendCommand(CmdBuf* cmdbuf, const T& command) {
+  cmdbuf->AppendCommand(&command, sizeof(command));
+}
+
+void Gfx9CmdWriter::InitializeLaunchTemplate() {
+  memset(&launch_template_, 0, sizeof(launch_template_));
+  GenerateCmdHeader(&launch_template_.indirect_buffer, IT_INDIRECT_BUFFER);
+}
+
+void Gfx9CmdWriter::InitializeAtomicTemplate() {
+  memset(&atomic_template_.atomic, 0, sizeof(atomic_template_));
+  GenerateCmdHeader(&atomic_template_.atomic, IT_ATOMIC_MEM);
+
+  // Specify the micro engine and cache policies
+  PM4MEC_ATOMIC_MEM* atomicCmd = &atomic_template_.atomic;
+  atomicCmd->bitfields2.cache_policy = cache_policy__mec_atomic_mem__stream;
+}
+
+void Gfx9CmdWriter::InitializeBarrierTemplate() {
+  memset(&pending_dispatch_template_, 0, sizeof(pending_dispatch_template_));
+  GenerateCmdHeader(&pending_dispatch_template_.event_write, IT_EVENT_WRITE);
+
+  MEC_EVENT_WRITE_event_index_enum index;
+  index = event_index__mec_event_write__cs_partial_flush;
+  pending_dispatch_template_.event_write.bitfields2.event_index = index;
+  pending_dispatch_template_.event_write.bitfields2.event_type = CS_PARTIAL_FLUSH;
+}
+
+void Gfx9CmdWriter::InitializeAcquireMemTemplate() {
+  memset(&invalidate_cache_template_, 0, sizeof(invalidate_cache_template_));
+  GenerateCmdHeader(&invalidate_cache_template_.acquire_mem, IT_ACQUIRE_MEM);
+
+  // Specify the CP module which will process this packet
+  PM4MEC_ACQUIRE_MEM* acquire_mem = &invalidate_cache_template_.acquire_mem;
+
+  // Specify the size of memory to invalidate. Size is
+  // specified in terms of 256 byte chunks. A coher_size
+  // of 0xFFFFFFFF actually specified 0xFFFFFFFF00 (40 bits)
+  // of memory. The field coher_size_hi specifies memory from
+  // bits 40-64 for a total of 256 TB.
+  acquire_mem->coher_size = 0xFFFFFFFF;
+  acquire_mem->bitfields4.coher_size_hi = 0xFFFFFF;
+
+  // Specify the address of memory to invalidate. The
+  // address must be 256 byte aligned.
+  acquire_mem->coher_base_lo = 0x00;
+  acquire_mem->bitfields6.coher_base_hi = 0x00;
+
+  // Specify the poll interval for determing if operation is complete
+  acquire_mem->bitfields7.poll_interval = 0x04;
+}
+
+void Gfx9CmdWriter::InitializeWaitRegMemTemplate() {
+  memset(&wait_reg_mem_template_, 0, sizeof(wait_reg_mem_template_));
+  GenerateCmdHeader(&wait_reg_mem_template_.wait_reg_mem, IT_WAIT_REG_MEM);
+
+  PM4MEC_WAIT_REG_MEM* wait_reg_mem = &wait_reg_mem_template_.wait_reg_mem;
+
+  wait_reg_mem->bitfields7.poll_interval = 0x04;
+  wait_reg_mem->bitfields2.operation = operation__mec_wait_reg_mem__wait_reg_mem;
+}
+
+void Gfx9CmdWriter::InitializeWriteDataTemplate(PM4MEC_WRITE_DATA* write_data, bool bit32) {
+  // Initialize the header of command packet by adjusting the
+  // size of payload - one 32bit DWord or two 32bit DWords
+  uint32_t cmd_size = (bit32) ? 1 : 2;
+  memset(write_data, 0, sizeof(PM4MEC_WRITE_DATA));
+  cmd_size = cmd_size + (sizeof(PM4MEC_WRITE_DATA) / sizeof(uint32_t));
+  write_data->ordinal1 = PM4_TYPE3_HDR(IT_WRITE_DATA, cmd_size);
+
+  // Set the bit to confirm the write operation and cache policy
+  write_data->bitfields2.wr_confirm = wr_confirm__mec_write_data__wait_for_write_confirmation;
+  write_data->bitfields2.cache_policy = cache_policy__mec_write_data__stream;
+
+  // Specify the command to increment address if writing more than one DWord
+  write_data->bitfields2.addr_incr = addr_incr__mec_write_data__increment_address;
+
+  // Specify the class to which the write destination belongs
+  write_data->bitfields2.dst_sel = dst_sel__mec_write_data__memory;
+}
+
+void Gfx9CmdWriter::InitializeWriteDataTemplate() {
+  InitializeWriteDataTemplate(&write_data_template_.write_data, true);
+}
+
+void Gfx9CmdWriter::InitializeWriteData64Template() {
+  InitializeWriteDataTemplate(&write_data64_template_.write_data, false);
+}
+
+void Gfx9CmdWriter::InitializeConditionalTemplate() {
+  /*
+  memset(&conditional_template_.conditional, 0, sizeof(conditional_template_));
+  GenerateCmdHeader(&conditional_template_.conditional, IT_COND_EXEC);
+
+  if (atc_support_) {
+    const uint32_t kAtcShift = 24;
+    conditional_template_.conditional.ordinal4 |= 1 << kAtcShift;
+  }
+  */
+}
+
+void Gfx9CmdWriter::InitializeEndOfKernelNotifyTemplate() {
+  memset(&notify_template_, 0, sizeof(notify_template_));
+  GenerateCmdHeader(&notify_template_.release_mem, IT_RELEASE_MEM);
+
+  // Set the event type to be bottom of pipe and cache policy
+  PM4MEC_RELEASE_MEM* rel_mem;
+  rel_mem = &notify_template_.release_mem;
+  rel_mem->bitfields2.event_type = BOTTOM_OF_PIPE_TS;
+  rel_mem->bitfields2.cache_policy = cache_policy__mec_release_mem__stream;
+  rel_mem->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe;
+
+  // Specify the attributes of source and destinations of data
+  rel_mem->bitfields3.int_sel = int_sel__mec_release_mem__none;
+  rel_mem->bitfields3.data_sel = data_sel__mec_release_mem__none;
+  rel_mem->bitfields3.dst_sel = dst_sel__mec_release_mem__memory_controller;
+}
+
+Gfx9CmdWriter::Gfx9CmdWriter(bool atc_support, bool pcie_atomic_support) {
+  // Initialize various state variables related to
+  // atomic operations and atc support
+  this->atc_support_ = atc_support;
+  this->pcie_atomic_support_ = pcie_atomic_support;
+
+  // Initialize various command templates
+  InitializeLaunchTemplate();
+  InitializeAtomicTemplate();
+  InitializeBarrierTemplate();
+  InitializeAcquireMemTemplate();
+  InitializeWaitRegMemTemplate();
+  InitializeWriteDataTemplate();
+  InitializeWriteData64Template();
+  InitializeConditionalTemplate();
+  InitializeEndOfKernelNotifyTemplate();
+}
+
+void Gfx9CmdWriter::BuildIndirectBufferCmd(CmdBuf* cmdbuf, const void* cmd_addr,
+                                           std::size_t cmd_size) {
+  // Verify the address is 4-byte aligned
+  uint64_t addr = uintptr_t(cmd_addr);
+  assert(!(addr & 0x3) && "IndirectBuffer address must be 4 byte aligned");
+
+  // Specify the address of indirect buffer encoding cmd stream
+  LaunchTemplate launch = launch_template_;
+
+  launch.indirect_buffer.bitfields2.ib_base_lo = (PtrLow32(cmd_addr) >> 2);
+  launch.indirect_buffer.ib_base_hi = PtrHigh32(cmd_addr);
+
+  // Specify the size of indirect buffer and cache policy to set
+  // upon executing the cmds of indirect buffer
+  launch.indirect_buffer.bitfields4.priv = 0;
+  launch.indirect_buffer.bitfields4.valid = 1;
+  launch.indirect_buffer.bitfields4.ib_size = cmd_size / sizeof(uint32_t);
+  launch.indirect_buffer.bitfields4.cache_policy = cache_policy__mec_indirect_buffer__stream;
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, launch);
+}
+
+void Gfx9CmdWriter::BuildAtomicPacket(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint32_t* addr,
+                                      uint32_t value, uint32_t compare) {
+  AtomicTemplate atomicTemplate = atomic_template_;
+  PM4MEC_ATOMIC_MEM* atomicCmd = &atomicTemplate.atomic;
+
+  // make sure the destination adddress is aligned
+  uint32_t address_low = PtrLow32((void*)addr);
+  uint32_t address_high = PtrHigh32((void*)addr);
+  assert(!(address_low & 0x7) && "destination address must be 8 byte aligned");
+  atomicCmd->addr_lo = address_low;
+  atomicCmd->addr_hi = address_high;
+
+  switch (atomic_op) {
+    case CommandWriter::kAtomicTypeIncrement:
+      assert(!(value != 0x01) && "Atomic Increment value should be 1");
+    case CommandWriter::kAtomicAdd:
+      atomicCmd->src_data_lo = value;
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_ADD_RTN_32;
+      break;
+    case CommandWriter::kAtomicTypeDecrement:
+      assert(!(value != 0x01) && "Atomic Decrement value should be 1");
+    case CommandWriter::kAtomicSubtract:
+      atomicCmd->src_data_lo = value;
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_SUB_RTN_32;
+      break;
+    case CommandWriter::kAtomicTypeBlockingCompareAndSwap:
+      atomicCmd->bitfields9.loop_interval = 128;
+      atomicCmd->bitfields2.command = command__mec_atomic_mem__loop_until_compare_satisfied;
+    case CommandWriter::kAtomicTypeCompareAndSwap:
+      atomicCmd->src_data_lo = value;
+      atomicCmd->cmp_data_lo = compare;
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_CMPSWAP_RTN_32;
+      break;
+    case CommandWriter::kAtomicSwap:
+      atomicCmd->src_data_lo = value;
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_SWAP_RTN_32;
+      break;
+    default:
+      assert((false) && "Atomic operation id is invalid");
+  }
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, atomicTemplate);
+}
+
+void Gfx9CmdWriter::BuildAtomicPacket64(CmdBuf* cmdbuf, AtomicType atomic_op,
+                                        volatile uint64_t* addr, uint64_t value, uint64_t compare) {
+  AtomicTemplate atomicTemplate = atomic_template_;
+  PM4MEC_ATOMIC_MEM* atomicCmd = &atomicTemplate.atomic;
+
+  // make sure the destination adddress is aligned
+  uint32_t address_low = PtrLow32((void*)addr);
+  uint32_t address_high = PtrHigh32((void*)addr);
+  assert(!(address_low & 0x7) && "destination address must be 8 byte aligned");
+  atomicCmd->addr_lo = address_low;
+  atomicCmd->addr_hi = address_high;
+
+  switch (atomic_op) {
+    case CommandWriter::kAtomicTypeIncrement:
+      assert(!(value != 0x01) && "Atomic Increment value should be 1");
+    case CommandWriter::kAtomicAdd:
+      atomicCmd->src_data_lo = Low32(value);
+      atomicCmd->src_data_hi = High32(value);
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_ADD_RTN_64;
+      break;
+    case CommandWriter::kAtomicTypeDecrement:
+      assert(!(value != 0x01) && "Atomic Decrement value should be 1");
+    case CommandWriter::kAtomicSubtract:
+      atomicCmd->src_data_lo = Low32(value);
+      atomicCmd->src_data_hi = High32(value);
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_SUB_RTN_64;
+      break;
+    case CommandWriter::kAtomicTypeBlockingCompareAndSwap:
+      atomicCmd->bitfields9.loop_interval = 128;
+      atomicCmd->bitfields2.command = command__mec_atomic_mem__loop_until_compare_satisfied;
+    case CommandWriter::kAtomicTypeCompareAndSwap:
+      atomicCmd->src_data_lo = Low32(value);
+      atomicCmd->src_data_hi = High32(value);
+      atomicCmd->cmp_data_lo = Low32(compare);
+      atomicCmd->cmp_data_hi = High32(compare);
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_CMPSWAP_RTN_64;
+      break;
+    case CommandWriter::kAtomicSwap:
+      atomicCmd->src_data_lo = Low32(value);
+      atomicCmd->src_data_hi = High32(value);
+      atomicCmd->bitfields2.atomic = TC_OP_ATOMIC_SWAP_RTN_64;
+      break;
+    default:
+      assert((false) && "Atomic operation id is invalid");
+  }
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, atomicTemplate);
+}
+
+void Gfx9CmdWriter::BuildBarrierCommand(CmdBuf* cmdBuf) {
+  APPEND_COMMAND_WRAPPER(cmdBuf, pending_dispatch_template_);
+}
+
+void Gfx9CmdWriter::BuildWriteDataCommand(CmdBuf* cmdbuf, uint32_t* write_addr,
+                                          uint32_t write_value) {
+  // Copy the initialized command packet and its payload
+  WriteDataTemplate command = write_data_template_;
+
+  // Encode the user specified address to write to
+  uint64_t addr = uintptr_t(write_addr);
+  assert(!(addr & 0x3) && "WriteData address must be 4 byte aligned");
+
+  // Specify the value to write
+  command.write_data_value = write_value;
+
+  // Test Code to see if this makes a difference
+  command.write_data.dst_mem_addr_hi = PtrHigh32(write_addr);
+  command.write_data.bitfields3c.dst_mem_addr_lo = (PtrLow32(write_addr) >> 2);
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, command);
+}
+
+void Gfx9CmdWriter::BuildWriteData64Command(CmdBuf* cmdbuf, uint64_t* write_addr,
+                                            uint64_t write_value) {
+  // Copy the initialized command packet and its payload
+  WriteData64Template command = write_data64_template_;
+
+  // Encode the user specified address to write to
+  uint64_t addr = uintptr_t(write_addr);
+  assert(!(addr & 0x3) && "WriteData address must be 4 byte aligned");
+
+  command.write_data.bitfields3c.dst_mem_addr_lo = (PtrLow32(write_addr) >> 2);
+  command.write_data.dst_mem_addr_hi = PtrHigh32(write_addr);
+
+  // Specify the value to write
+  command.write_data_value = write_value;
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, command);
+}
+
+void Gfx9CmdWriter::BuildWaitRegMemCommand(CmdBuf* cmdbuf, bool mem_space, uint64_t wait_addr,
+                                           bool func_eq, uint32_t mask_val, uint32_t wait_val) {
+  WaitRegMemTemplate wait_cmd = wait_reg_mem_template_;
+
+  // Apply the space to which addr belongs
+  if (mem_space) {
+    wait_cmd.wait_reg_mem.bitfields2.mem_space = mem_space__mec_wait_reg_mem__memory_space;
+  } else {
+    wait_cmd.wait_reg_mem.bitfields2.mem_space = mem_space__mec_wait_reg_mem__register_space;
+  }
+
+  // Apply the function - equal / not equal desired by user
+  if (func_eq) {
+    wait_cmd.wait_reg_mem.bitfields2.function =
+        function__mec_wait_reg_mem__equal_to_the_reference_value;
+  } else {
+    wait_cmd.wait_reg_mem.bitfields2.function =
+        function__mec_wait_reg_mem__not_equal_reference_value;
+  }
+
+  // Value to use in applying equal / not equal function
+  wait_cmd.wait_reg_mem.reference = wait_val;
+
+  // Apply the mask on value at address/register
+  wait_cmd.wait_reg_mem.mask = mask_val;
+
+  // The address to poll should be DWord (4 byte) aligned
+  // Update upper 32 bit address if addr is not a register
+  if (mem_space) {
+    assert(!(wait_addr & 0x3) && "WaitRegMem address must be 4 byte aligned");
+  }
+  wait_cmd.wait_reg_mem.bitfields3a.mem_poll_addr_lo = (Low32(wait_addr) >> 2);
+  if (mem_space) {
+    wait_cmd.wait_reg_mem.mem_poll_addr_hi = High32(wait_addr);
+  }
+
+  // Append the command to cmd stream
+  APPEND_COMMAND_WRAPPER(cmdbuf, wait_cmd);
+}
+
+void Gfx9CmdWriter::BuildConditionalExecute(CmdBuf* cmdbuf, uint32_t* signal, uint16_t count) {
+  assert(false && "BuildConditionalExecute method is not implemented");
+  /*
+  ConditionalExecuteTemplate conditional = conditional_template_;
+
+  uint32_t address_low = PtrLow32(signal);
+  uint32_t address_high = PtrHigh32(signal);
+  assert(!(address_low & 0x7) && "destination address must be 8 byte aligned");
+
+  conditional.conditional.boolAddrLo = address_low;
+  conditional.conditional.boolAddrHi = address_high;
+  conditional.conditional.execCount = count;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, conditional);
+  */
+}
+
+void Gfx9CmdWriter::BuildUpdateHostAddress(CmdBuf* cmdbuf, uint64_t* addr, int64_t value) {
+  // If Atomics are supported, use it
+  if (pcie_atomic_support_) {
+    BuildAtomicPacket64(cmdbuf, CommandWriter::AtomicType::kAtomicSwap, (volatile uint64_t*)addr,
+                        value);
+    return;
+  }
+
+  BuildWriteData64Command(cmdbuf, addr, value);
+  return;
+}
+
+void Gfx9CmdWriter::BuildBOPNotifyCmd(CmdBuf* cmdbuf, const void* write_addr, uint32_t write_value,
+                                      bool interrupt) {
+  // Initialize the command including its header
+  EndofKernelNotifyTemplate eop = notify_template_;
+  PM4MEC_RELEASE_MEM* rel_mem = &eop.release_mem;
+
+  // Program CP to perform various cache operations
+  // before issuing the write operation commences
+  rel_mem->bitfields2.tc_action_ena = true;
+  rel_mem->bitfields2.tc_wb_action_ena = true;
+
+  // Update cmd to write a user specified 32-bit value
+  rel_mem->data_lo = write_value;
+  rel_mem->bitfields3.data_sel = data_sel__mec_release_mem__send_32_bit_low;
+
+  // Update cmd with user specified address to write to
+  rel_mem->address_hi = High32(uint64_t(write_addr));
+  rel_mem->bitfields4b.address_lo_64b = (Low32(uint64_t(write_addr) >> 3));
+
+  // Update cmd to issue interrupt if user has requested it
+  if (interrupt) {
+    rel_mem->bitfields3.int_sel = int_sel__mec_release_mem__send_interrupt_after_write_confirm;
+  }
+
+  // Serialize the command as stream of Dwords
+  APPEND_COMMAND_WRAPPER(cmdbuf, eop);
+}
+
+void Gfx9CmdWriter::BuildBarrierFenceCommands(CmdBuf* cmdbuf) {
+  // TODO: temporarily remove the check because some OpenCL tests
+  // (test_buffers, test_relationals) are failing.
+  //    if (using_cc_memory_policy_)
+  //        return;
+  AcquireMemTemplate invalidate_src_caches = invalidate_cache_template_;
+
+  // wbINVL2 by default writes-back and invalidates both L1 and L2
+  invalidate_src_caches.acquire_mem.bitfields2.coher_cntl = CP_COHER_CNTL__TC_ACTION_ENA_MASK;
+  invalidate_src_caches.acquire_mem.bitfields2.coher_cntl |= CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, invalidate_src_caches);
+}
+
+/*
+// PM4 packet for profilers
+#define PM4_PACKET3 (0xC0000000)
+#define PM4_PACKET3_CMD_SHIFT 8
+#define PM4_PACKET3_COUNT_SHIFT 16
+
+#define PACKET3(cmd, count)                                 \
+  (PM4_PACKET3 | (((count)-1) << PM4_PACKET3_COUNT_SHIFT) | \
+   ((cmd) << PM4_PACKET3_CMD_SHIFT))
+*/
+
+// Structure to store the event PM4 packet
+typedef struct WriteRegPacket_ { uint32_t item[3]; } WriteRegPacket;
+
+void Gfx9CmdWriter::BuildWriteEventPacket(CmdBuf* cmdbuf, uint32_t event) {
+  PM4MEC_EVENT_WRITE cp_event_initiator;
+  memset(&cp_event_initiator, 0, sizeof(PM4MEC_EVENT_WRITE));
+  cp_event_initiator.ordinal1 =
+      PM4_TYPE3_HDR(IT_EVENT_WRITE, (sizeof(PM4MEC_EVENT_WRITE) / sizeof(uint32_t)));
+  cp_event_initiator.ordinal2 = 0;
+
+  VGT_EVENT_TYPE eventType = Reserved_0x00;
+  switch (event) {
+    case kPerfCntrsStart:
+      eventType = PERFCOUNTER_START;
+      break;
+    case kPerfCntrsStop:
+      eventType = PERFCOUNTER_STOP;
+      break;
+    case kPerfCntrsSample:
+      eventType = PERFCOUNTER_SAMPLE;
+      break;
+    default:
+      assert(false && "Illegal VGT Event Id");
+  }
+
+  MEC_EVENT_WRITE_event_index_enum index;
+  index = event_index__mec_event_write__other;
+  cp_event_initiator.bitfields2.event_index = index;
+  cp_event_initiator.bitfields2.event_type = eventType;
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, cp_event_initiator);
+}
+
+void Gfx9CmdWriter::BuildWriteUnshadowRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] =
+      PM4_TYPE3_HDR(IT_SET_UCONFIG_REG, (1 + sizeof(PM4MEC_SET_CONFIG_REG) / sizeof(uint32_t)));
+  packet.item[1] = (addr - UCONFIG_SPACE_START);
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+}
+
+void Gfx9CmdWriter::BuildWriteUConfigRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] =
+      PM4_TYPE3_HDR(IT_SET_UCONFIG_REG, (1 + sizeof(PM4MEC_SET_CONFIG_REG) / sizeof(uint32_t)));
+  packet.item[1] = (addr - UCONFIG_SPACE_START);
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+}
+
+void Gfx9CmdWriter::BuildWriteShRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  WriteRegPacket packet;
+  packet.item[0] =
+      PM4_TYPE3_HDR(IT_SET_SH_REG, (1 + sizeof(PM4MEC_SET_CONFIG_REG) / sizeof(uint32_t)));
+  packet.item[1] = (addr - PERSISTENT_SPACE_START);
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+}
+
+void Gfx9CmdWriter::BuildCopyDataPacket(CmdBuf* cmdbuf, uint32_t src_sel, uint32_t src_addr_lo,
+                                        uint32_t src_addr_hi, uint32_t* dst_addr, uint32_t size,
+                                        bool wait) {
+  PM4MEC_COPY_DATA cmd_data;
+  memset(&cmd_data, 0, sizeof(PM4MEC_COPY_DATA));
+  cmd_data.ordinal1 = PM4_TYPE3_HDR(IT_COPY_DATA, (sizeof(PM4MEC_COPY_DATA) / sizeof(uint32_t)));
+
+  MEC_COPY_DATA_src_sel_enum data_src = src_sel__mec_copy_data__memory;
+  switch (src_sel) {
+    case 0:
+      data_src = src_sel__mec_copy_data__mem_mapped_register;
+      break;
+    case 4:
+      data_src = src_sel__mec_copy_data__perfcounters;
+      break;
+    default:
+      assert(false && "CopyData Illegal value for source of data");
+      break;
+  }
+  cmd_data.bitfields2.src_sel = data_src;
+  cmd_data.bitfields2.src_cache_policy = src_cache_policy__mec_copy_data__stream;
+
+  cmd_data.bitfields2.dst_sel = dst_sel__mec_copy_data__memory;
+  cmd_data.bitfields2.dst_cache_policy = dst_cache_policy__mec_copy_data__stream;
+
+  cmd_data.bitfields2.wr_confirm = (MEC_COPY_DATA_wr_confirm_enum)wait;
+  cmd_data.bitfields2.count_sel = (size == 0) ? count_sel__mec_copy_data__32_bits_of_data
+                                              : count_sel__mec_copy_data__64_bits_of_data;
+
+  // Specify the source register offset
+  cmd_data.bitfields3a.src_reg_offset = src_addr_lo;
+
+  // Specify the destination memory address
+  cmd_data.dst_addr_hi = PtrHigh32(dst_addr);
+  if (size == 0) {
+    cmd_data.bitfields5b.dst_32b_addr_lo = (PtrLow32(dst_addr) >> 2);
+  } else {
+    cmd_data.bitfields5c.dst_64b_addr_lo = (PtrLow32(dst_addr) >> 3);
+  }
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, cmd_data);
+}
+
+void Gfx9CmdWriter::BuildCacheFlushPacket(CmdBuf* cmdbuf) {
+  // Initialize the command header
+  PM4MEC_ACQUIRE_MEM cache_flush = invalidate_cache_template_.acquire_mem;
+
+  // Program Coherence Control Register. Initialize L2 Cache flush
+  // for Non-Coherent memory blocks
+  uint32_t coher_cntl = 0;
+
+  coher_cntl |= CP_COHER_CNTL__TC_ACTION_ENA_MASK;
+  coher_cntl |= CP_COHER_CNTL__TCL1_ACTION_ENA_MASK;
+  coher_cntl |= CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK;
+  coher_cntl |= CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK;
+  coher_cntl |= CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK;
+  cache_flush.bitfields2.coher_cntl = coher_cntl;
+
+  // Copy AcquireMem command buffer stream
+  APPEND_COMMAND_WRAPPER(cmdbuf, cache_flush);
+}
+
+void Gfx9CmdWriter::BuildWriteWaitIdlePacket(CmdBuf* cmdbuf) {
+  BuildBarrierCommand(cmdbuf);
+  BuildCacheFlushPacket(cmdbuf);
+}
+
+// Will issue a VGT event including a cache flush later on
+void Gfx9CmdWriter::BuildVgtEventPacket(CmdBuf* cmdbuf, uint32_t vgtEvent) {
+  PM4MEC_EVENT_WRITE cp_event_initiator;
+  memset(&cp_event_initiator, 0, sizeof(PM4MEC_EVENT_WRITE));
+  cp_event_initiator.ordinal1 =
+      PM4_TYPE3_HDR(IT_EVENT_WRITE, (sizeof(PM4MEC_EVENT_WRITE) / sizeof(uint32_t)));
+  cp_event_initiator.ordinal2 = 0;
+
+  VGT_EVENT_TYPE eventType = Reserved_0x00;
+  switch (vgtEvent) {
+    case kPerfCntrsStart:
+      eventType = PERFCOUNTER_START;
+      break;
+    case kPerfCntrsStop:
+      eventType = PERFCOUNTER_STOP;
+      break;
+    case kPerfCntrsSample:
+      eventType = PERFCOUNTER_SAMPLE;
+      break;
+    case kThrdTraceStart:
+      eventType = THREAD_TRACE_START;
+      break;
+    case kThrdTraceStop:
+      eventType = THREAD_TRACE_STOP;
+      break;
+    case kThrdTraceFlush:
+      eventType = THREAD_TRACE_FLUSH;
+      break;
+    case kThrdTraceFinish:
+      eventType = THREAD_TRACE_FINISH;
+      break;
+    default:
+      assert(false && "Illegal VGT Event Id");
+  }
+
+  MEC_EVENT_WRITE_event_index_enum index;
+  index = event_index__mec_event_write__other;
+  cp_event_initiator.bitfields2.event_index = index;
+  cp_event_initiator.bitfields2.event_type = eventType;
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, cp_event_initiator);
+
+  // Check If I should be issuing a cache flush operation as well
+  // test and remove it
+  BuildCacheFlushPacket(cmdbuf);
+}
+
+void Gfx9CmdWriter::BuildWriteRegisterPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value) {
+  /*
+  WriteRegPacket packet;
+  packet.item[0] = (PM4_TYPE3_HDR(
+      IT_SET_CONFIG_REG, 1 + PM4_CMD_SET_CONFIG_REG_DWORDS, ShaderGraphics, 0));
+  packet.item[1] = addr - CONFIG_SPACE_START;
+  packet.item[2] = value;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, packet);
+
+  return;
+  */
+}
+
+void Gfx9CmdWriter::BuildWriteEventQueryPacket(CmdBuf* cmdbuf, uint32_t event, uint32_t* addr) {
+  PM4MEC_EVENT_WRITE_QUERY cp_event_initiator;
+  memset(&cp_event_initiator, 0, sizeof(PM4MEC_EVENT_WRITE_QUERY));
+  cp_event_initiator.ordinal1 =
+      PM4_TYPE3_HDR(IT_EVENT_WRITE, (sizeof(PM4MEC_EVENT_WRITE_QUERY) / sizeof(uint32_t)));
+  cp_event_initiator.ordinal2 = 0;
+
+  // Update switch statements you want to support
+  VGT_EVENT_TYPE eventType = Reserved_0x00;
+  switch (event) {
+    default:
+      assert(false && "Illegal VGT Event Id");
+  }
+
+  MEC_EVENT_WRITE_event_index_enum index;
+  cp_event_initiator.bitfields2.event_type = eventType;
+  index = (MEC_EVENT_WRITE_event_index_enum)EventTypeToIndexTable[eventType];
+  cp_event_initiator.bitfields2.event_index = index;
+
+  // set the address
+  uint32_t addrLo = PtrLow32(addr);
+  uint32_t addrHi = PtrHigh32(addr);
+  ((addrLo & 0x7) != 0) ? assert(false) : assert(true);
+
+  cp_event_initiator.address_hi = addrHi;
+  cp_event_initiator.bitfields3.address_lo = (addrLo >> 3);
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, cp_event_initiator);
+}
+
+size_t Gfx9CmdWriter::SizeOfAtomicPacket() const {
+  return sizeof(AtomicTemplate) / sizeof(uint32_t);
+}
+
+void Gfx9CmdWriter::BuildFlushCacheCmd(CmdBuf* cmdbuf, FlushCacheOptions* options,
+                                       uint32_t* writeAddr, uint32_t writeVal) {
+  PM4MEC_ACQUIRE_MEM cache_flush = invalidate_cache_template_.acquire_mem;
+
+  // Verify write back address is valid. Note that this address is NOT
+  // used on CI. But to have a same interface as that on SI, we keep
+  // the address argument in this function. Thus, this check always pass
+  // no matter the address is NULL or not.
+  (writeAddr == NULL) ? assert(true) : assert(true);
+
+  // Program Coherence Control Register. Initialize L2 Cache flush
+  // for Non-Coherent memory blocks
+  uint32_t coher_cntl = 0;
+  coher_cntl |= (options->l1) ? CP_COHER_CNTL__TCL1_ACTION_ENA_MASK : 0;
+  coher_cntl |= (options->l2)
+      ? (CP_COHER_CNTL__TC_ACTION_ENA_MASK | CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK)
+      : 0;
+  coher_cntl |= (options->icache) ? CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK : 0;
+  coher_cntl |= (options->kcache) ? CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK : 0;
+  cache_flush.bitfields2.coher_cntl = coher_cntl;
+
+  // Append the built command into output Command Buffer
+  APPEND_COMMAND_WRAPPER(cmdbuf, cache_flush);
+  return;
+}
+
+void Gfx9CmdWriter::BuildDmaDataPacket(CmdBuf* cmdbuf, uint32_t* srcAddr, uint32_t* dstAddr,
+                                       uint32_t copySize, bool waitForConfirm) {
+  PM4MEC_DMA_DATA cmdDmaData;
+  memset(&cmdDmaData, 0, sizeof(PM4MEC_DMA_DATA));
+  cmdDmaData.header.u32All =
+      PM4_TYPE3_HDR(IT_DMA_DATA, (sizeof(PM4MEC_DMA_DATA) / sizeof(uint32_t)));
+
+  // Specify attributes of source buffer such as its
+  // location and Cache policy
+  cmdDmaData.bitfields2.src_sel = src_sel__mec_dma_data__src_addr_using_sas;
+  cmdDmaData.bitfields2.src_cache_policy = src_cache_policy__mec_dma_data__stream;
+
+  // Specify attributes of destination buffer such as its
+  // location and Cache policy
+  cmdDmaData.bitfields2.dst_sel = dst_sel__mec_dma_data__dst_addr_using_das;
+  cmdDmaData.bitfields2.dst_cache_policy = dst_cache_policy__mec_dma_data__stream;
+
+  // Specify the source and destination addr
+  cmdDmaData.src_addr_lo_or_data = PtrLow32(srcAddr);
+  cmdDmaData.src_addr_hi = PtrHigh32(srcAddr);
+  cmdDmaData.dst_addr_lo = PtrLow32(dstAddr);
+  cmdDmaData.dst_addr_hi = PtrHigh32(dstAddr);
+
+  // Number of bytes to copy. The command restricts
+  // the size to be (64 MB - 1) - 26 Bits
+  assert(copySize < 0x1FFFFF);
+  cmdDmaData.bitfields7.byte_count = copySize;
+
+  // Indicate that DMA Cmd should wait if its source
+  // is the destination of a previous DMA Cmd
+  cmdDmaData.bitfields7.raw_wait = waitForConfirm;
+
+  APPEND_COMMAND_WRAPPER(cmdbuf, cmdDmaData);
+  return;
+}
+
+
+}  // gfx9 namespace
+
+}  // pm4_profile

projects/rocr-runtime/runtime/hsa-amd-aqlprofile/src/commandwriter/gfx9_cmdwriter.h

@@ -0,0 +1,199 @@
+#ifndef _GFX9_CMDWRITER_H_
+#define _GFX9_CMDWRITER_H_
+
+#include "cmdwriter.h"
+#include "gfx9_cmds.h"
+
+namespace pm4_profile {
+
+namespace gfx9 {
+
+
+/// @brief class Gfx9CmdWriter implements the virtual class CommandWriter
+/// for GFX9 chipsets
+class Gfx9CmdWriter : public CommandWriter {
+ public:
+  Gfx9CmdWriter(bool atc_support, bool pcie_atomic_support);
+
+  /// @brief Dword specifying NOOP command for GFX9 chipsets. The macro
+  /// populates the NOOP command which is 32-bits wide. The second parameter,
+  /// the COUNT field of NOOP command, specifies the number of Dwords to skip.
+  /// To skip ZERO Dwords the value should be set to 0x3FFF. Since the macro
+  /// decrements the second parameter by TWO, an artifact of its definition,
+  /// the value is incremented by TWO to 0x4001 (0x3FFF + 2).
+  ///
+  inline uint32_t GetNoOpCmd() {
+    static const uint32_t nopCmd = PM4_TYPE3_HDR(IT_NOP, 0x4001);
+    return nopCmd;
+  }
+
+  void BuildBarrierCommand(CmdBuf* cmdBuf);
+
+  void BuildIndirectBufferCmd(CmdBuf* cmdbuf, const void* cmd_addr, std::size_t cmd_size);
+
+  void BuildBOPNotifyCmd(CmdBuf* cmdbuf, const void* write_addr, uint32_t write_val,
+                         bool interrupt);
+
+  void BuildBarrierFenceCommands(CmdBuf* cmdbuf);
+
+  void BuildWriteEventPacket(CmdBuf* cmdbuf, uint32_t event);
+
+  void BuildWaitRegMemCommand(CmdBuf* cmdbuf, bool mem_space, uint64_t wait_addr, bool func_eq,
+                              uint32_t mask_val, uint32_t wait_val);
+
+  void BuildWriteUnshadowRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  /// @brief Build CP command to program a Gpu register
+  ///
+  /// @param cmdbuf Pointer to command buffer to be appended
+  /// @param addr Register to be programmed
+  /// @param value Value to write into register
+  ///
+  /// @return void
+  void BuildWriteUConfigRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  void BuildWriteShRegPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  void BuildCopyDataPacket(CmdBuf* cmdbuf, uint32_t src_sel, uint32_t src_addr_lo,
+                           uint32_t src_addr_hi, uint32_t* dst_addr, uint32_t size, bool wait);
+
+  void BuildWriteWaitIdlePacket(CmdBuf* cmdbuf);
+
+  // Will issue a VGT event including a cache flush later on
+  void BuildVgtEventPacket(CmdBuf* cmdbuf, uint32_t vgtEvent);
+
+  void BuildWriteRegisterPacket(CmdBuf* cmdbuf, uint32_t addr, uint32_t value);
+
+  void BuildWriteEventQueryPacket(CmdBuf* cmdbuf, uint32_t event, uint32_t* addr);
+
+  void BuildAtomicPacket(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint32_t* addr,
+                         uint32_t value, uint32_t compare);
+
+  void BuildAtomicPacket64(CmdBuf* cmdbuf, AtomicType atomic_op, volatile uint64_t* addr,
+                           uint64_t value = 0, uint64_t compare = 0);
+
+  size_t SizeOfAtomicPacket() const;
+
+  void BuildConditionalExecute(CmdBuf* cmdbuf, uint32_t* signal, uint16_t count);
+
+  void BuildWriteDataCommand(CmdBuf* cmdbuf, uint32_t* write_addr, uint32_t write_value);
+
+  void BuildWriteData64Command(CmdBuf* cmdbuf, uint64_t* write_addr, uint64_t write_value);
+
+  void BuildCacheFlushPacket(CmdBuf* cmdbuf);
+
+  /// Writes into input buffer Gpu commands to flush its cache. It is
+  /// necessary that the buffer provided for flush commands is large
+  /// enough to accommodate the full set of commands. It should be at
+  /// least 512 bytes.
+  ///
+  /// @param tsCmdBuf Buffer to write commands to.
+  /// @param writeAddr Registered address into which GPU should write
+  /// a user provided value upon executing the flush commands.
+  /// @param writeVal User provided value written by GPU at user provided
+  /// address, upon executing the flush commands.
+  ///
+  /// @return void
+  void BuildFlushCacheCmd(CmdBuf* cmdBuf, FlushCacheOptions* options, uint32_t* writeAddr,
+                          uint32_t writeVal);
+
+  /// Builds Gpu command to copy data from source to destination buffer
+  /// using DMA engine.
+  ///
+  /// @param cmdbuf Buffer updated with Gpu copy command
+  /// @param srcAddr Address of source buffer address
+  /// @param dstAddr Address of destination buffer address
+  /// @param copySize Size of data to copy in bytes
+  /// @param waitForCompletion if command should wait for copying to complete
+  void BuildDmaDataPacket(CmdBuf* cmdBuf, uint32_t* srcAddr, uint32_t* dstAddr, uint32_t copySize,
+                          bool waitForCompletion);
+
+ protected:
+  /// @brief Append an instance of Gpu command into input command buffer stream.
+  ///
+  /// @param cmdbuf CommandWriter object appended with anohter Gpu command
+  ///
+  /// @param cmd Gpu command to be appended into command buffer
+  ///
+  /// @return void
+  template <class T> void AppendCommand(CmdBuf* cmdbuf, const T& cmd);
+
+ private:
+  /// @brief Initializes a Gpu command which can be used to
+  /// reference a Gpu command stream indirectly
+  void InitializeLaunchTemplate();
+
+  /// @brief Initializes a Gpu command which can be used to
+  /// flush Gpu caches and write to a user configurable address
+  /// to indicate an end of kernel
+  void InitializeEndOfKernelNotifyTemplate();
+
+  /// @brief Initializes a Gpu command to perform atomic operations
+  ////
+  void InitializeAtomicTemplate();
+
+  /// @brief Initializes a Gpu command to allow conditional execution
+  /// of a Gpu command stream
+  void InitializeConditionalTemplate();
+
+  /// @brief Initializes a Gpu command to let command processor
+  /// wait for some update before letting other commands to be
+  /// processed
+  void InitializeWaitRegMemTemplate();
+
+  /// @brief Initializes the template for Barrier command.
+  /// Applications can use Barrier command to ensure their
+  /// command is executed only after all other commands have
+  /// completed their execution.
+  void InitializeBarrierTemplate();
+
+  void BuildUpdateHostAddress(CmdBuf* cmdbuf, uint64_t* addr, int64_t value);
+
+  /// @brief Initializes Acquire Memory command template. Users
+  /// can submit this command to invalidate Gpu caches - L1 and
+  /// or L2.
+  void InitializeAcquireMemTemplate();
+
+  /// @brief Initializes an instance of Write Data command
+  /// for use by an application
+  void InitializeWriteDataTemplate();
+  void InitializeWriteData64Template();
+  void InitializeWriteDataTemplate(PM4MEC_WRITE_DATA* write_data, bool bit32);
+
+  /// @brief Builds wait_reg_mem with EQUALS condition
+  void BuildWaitRegMemCommand(CmdBuf* cmdbuf, uint64_t wait_addr, uint32_t wait_value);
+
+  /// @brief Instance of Gpu command to reference dispatch commands
+  LaunchTemplate launch_template_;
+
+  /// @brief Instance of Gpu command to use in determing end of kernel
+  EndofKernelNotifyTemplate notify_template_;
+
+  /// @brief Instance of Gpu command to use in performing atomic operations
+  AtomicTemplate atomic_template_;
+
+  /// @brief Instance of Pm4 command WRITE_DATA
+  WriteDataTemplate write_data_template_;
+  WriteData64Template write_data64_template_;
+
+  /// @brief Instance of Pm4 command EVENT_WRITE
+  BarrierTemplate pending_dispatch_template_;
+
+  /// @brief Instance of Pm4 command ACQUIRE_MEM
+  AcquireMemTemplate invalidate_cache_template_;
+
+  /// @brief Instance of Pm4 command WAIT_REG_MEM
+  WaitRegMemTemplate wait_reg_mem_template_;
+
+  /// @brief ATC support.
+  bool atc_support_;
+
+  /// @brief PCIe atomic support.
+  bool pcie_atomic_support_;
+};
+
+}  // gfx9
+
+}  // pm4_profile
+
+#endif  //  _GFX9_CMDWRITER_H_