Add 'projects/clr/' from commit 'ed903e888949f3631f133847f834b06b817b63b8'

git-subtree-dir: projects/clr
git-subtree-mainline: 840ad49d28
git-subtree-split: ed903e8889

projects/clr/.azuredevops/rocm-ci.yml

@@ -0,0 +1,71 @@
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: matching_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/HIP
+    ref: $(Build.SourceBranch)
+  - repository: hipother_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/hipother
+    ref: $(Build.SourceBranch)
+  pipelines:
+  - pipeline: hip_pipeline
+    source: \HIP
+    trigger:
+      branches:
+        include:
+        - amd-staging
+        - amd-mainline
+  - pipeline: hipother_pipeline
+    source: \hipother
+    trigger:
+      branches:
+        include:
+        - amd-staging
+        - amd-mainline
+
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml@pipelines_repo
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - amd-staging
+    - amd-mainline
+  paths:
+    exclude:
+    - CODEOWNERS
+    - LICENCE
+    - '**/*.md'
+
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - amd-staging
+    - amd-mainline
+  paths:
+    exclude:
+    - CODEOWNERS
+    - LICENCE
+    - '**/*.md'
+  drafts: false
+
+jobs:
+# if the build reason is a resource trigger, it means trigger is HIP or hipother repo build
+# HIP/hipother repo build would have just built runtime, just copy their build products
+# this is to ensure clr has latest good package for combined-packaging jobs
+# combined-packaging jobs only have to look at clr pipeline for latest runtime
+# to remove logic of comparing build products from both clr, hip, hipother triggers
+  - ${{ if eq(variables['Build.Reason'], 'ResourceTrigger') }}:
+    - template: ${{ variables.CI_COMPONENT_PATH }}/copyHIP.yml@pipelines_repo
+  - ${{ if ne(variables['Build.Reason'], 'ResourceTrigger') }}:
+    - template: ${{ variables.CI_COMPONENT_PATH }}/HIP.yml@pipelines_repo

projects/clr/.clang-format

@@ -0,0 +1,10 @@
+Language: Cpp
+BasedOnStyle: Google
+AlignEscapedNewlinesLeft: false
+AlignOperands: false
+ColumnLimit: 100
+AlwaysBreakTemplateDeclarations: false
+DerivePointerAlignment: false
+IndentFunctionDeclarationAfterType: false
+MaxEmptyLinesToKeep: 2
+SortIncludes: false

projects/clr/.gitattributes

@@ -0,0 +1,20 @@
+# Set the default behavior, in case people don't have core.autolf set.
+* text=auto
+
+# Explicitly declare text files you want to always be normalized and converted
+# to have LF line endings on checkout.
+*.c text eol=lf
+*.cpp text eol=lf
+*.cc text eol=lf
+*.h text eol=lf
+*.hpp text eol=lf
+*.txt text eol=lf
+
+# Define files to support auto-remove trailing white space
+# Need to run the command below, before add modified file(s) to the staging area
+# git config filter.trimspace.clean 'sed -e "s/[[:space:]]*$//g"'
+*.cpp filter=trimspace
+*.c filter=trimspace
+*.h filter=trimspacecpp
+*.hpp filter=trimspace
+*.md filter=trimspace

projects/clr/.github/hooks/clang-format-check.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+RANGE=""
+
+while [[ $# -gt 0 ]]; do
+  echo $1
+  echo $2
+  case "$1" in
+  --range)
+    RANGE="$2"
+    shift 2
+    ;;
+  *)
+    echo "Unknown arg $1" >&2
+    exit 64
+    ;;
+  esac
+done
+
+regex='\.(c|cc|cpp|cxx|h|hh|hpp|hxx)$'
+
+if [[ -n $RANGE ]]; then
+  files=$(git diff --name-only "$RANGE" | grep -E "$regex" || true)
+else
+  files=$(git diff --cached --name-only --diff-filter=ACMR | grep -E "$regex" || true)
+fi
+echo "Checking $files"
+[[ -z $files ]] && exit 0
+
+clang_bin="${CLANG_FORMAT:-clang-format}"
+if ! command -v "$clang_bin" >/dev/null 2>&1; then
+  if [[ -x "/c/Program Files/LLVM/bin/clang-format.exe" ]]; then
+    clang_bin="/c/Program Files/LLVM/bin/clang-format.exe"
+  fi
+fi
+
+clang_format_diff="${CLANG_FORMAT_DIFF:-clang-format-diff}"
+if ! command -v "$clang_format_diff" >/dev/null 2>&1; then
+  if [[ -x "/c/Program Files/LLVM/share/clang/clang-format-diff.py" ]]; then
+    clang_format_diff="/c/Program Files/LLVM/share/clang/clang-format-diff.py"
+  fi
+fi
+
+for file in $files; do
+  echo "Checking lines of $file"
+
+  if [[ -n $RANGE ]]; then
+    diff_output=$(git diff -U0 "$RANGE" -- "$file")
+  else
+    diff_output=$(git diff -U0 --cached -- "$file")
+  fi
+
+  echo "$diff_output" | "$clang_format_diff" -style=file -fallback-style=none -p1
+done

projects/clr/.github/hooks/pre-commit

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+exec "$(git rev-parse --show-toplevel)/.github/hooks/clang-format-check.sh"

projects/clr/.github/palamida.yml

@@ -0,0 +1,5 @@
+disabled: false
+scmId: gh-emu-rocm
+branchesToScan:
+  - amd-staging
+  - amd-mainline

projects/clr/.github/pull_request_template.md

@@ -0,0 +1,36 @@
+## Associated JIRA ticket number/Github issue number
+<!-- For example: "Closes #1234" or "Fixes SWDEV-123456" -->
+
+## What type of PR is this? (check all applicable)
+
+- [ ] Refactor
+- [ ] Feature
+- [ ] Bug Fix
+- [ ] Optimization
+- [ ] Documentation Update
+- [ ] Continuous Integration
+
+## What were the changes?
+
+<!-- Please give a short summary of the change. -->
+
+## Why are these changes needed?
+
+<!-- Please explain the motivation behind the change and why this solves the given problem. -->
+
+## Updated CHANGELOG?
+
+<!-- Needed for Release updates for a ROCm release. -->
+
+- [ ] Yes
+- [ ] No, Does not apply to this PR.
+
+## Added/Updated documentation?
+
+- [ ] Yes
+- [ ] No, Does not apply to this PR.
+
+## Additional Checks
+
+- [ ] I have added tests relevant to the introduced functionality, and the unit tests are passing locally.
+- [ ] Any dependent changes have been merged.

projects/clr/.github/scripts/validate_pr_description.py

@@ -0,0 +1,76 @@
+import os, re, sys
+from typing import List, Optional
+
+
+def is_checkbox(line: str) -> bool:
+    return bool(re.match(r"^\s*-\s*\[[ xX]\]\s*.+", line))
+
+
+def is_checked(line: str) -> bool:
+    return bool(re.match(r"^\s*-\s*\[\s*[xX]\s*\]\s*.+", line))
+
+
+def is_comment(line: str) -> bool:
+    return bool(re.match(r"^\s*<!--.*-->\s*$", line))
+
+
+def text_clean(lines: List[str]) -> str:
+    text = [line for line in lines if not is_comment(line)]
+    return "".join("".join(text).strip().split())
+
+
+def validate_section(section_name: str, lines: List[str]) -> Optional[str]:
+    has_checkboxes = any(is_checkbox(line) for line in lines)
+    if has_checkboxes:
+        if not any(is_checked(line) for line in lines):
+            return f"Section {section_name} is a checklist without selections"
+        return None
+    if not text_clean(lines):
+        return f"Section {section_name} is empty text section"
+    return None
+
+
+def check_description(description: str) -> List[str]:
+    if not description:
+        # pull_request_template is not merged yet, so treat as valid for now
+        return []
+        # return ["PR description is empty"]
+
+    sections = []
+    current_section = None
+    current_lines = []
+    errors = []
+
+    for line in description.splitlines():
+        header_match = re.match(r"^\s*##\s*(.+?)\s*$", line)
+        if header_match:
+            if current_section:
+                sections.append((current_section, current_lines))
+            current_section = header_match.group(1)
+            current_lines = []
+        elif current_section:
+            current_lines.append(line)
+
+    if current_section:
+        sections.append((current_section, current_lines))
+
+    if not sections:
+        return ["No sections available, template is empty"]
+
+    for section_name, section_lines in sections:
+        error = validate_section(section_name, section_lines)
+        if error:
+            errors.append(error)
+
+    return errors
+
+
+if __name__ == "__main__":
+    pr_description = os.getenv("PR_DESCRIPTION", "")
+
+    errors = check_description(pr_description)
+    if not errors:
+        print("All good")
+        exit(0)
+    print("\n".join(errors))
+    exit(1)

projects/clr/.github/workflows/ai_codeql_fix.yaml

@@ -0,0 +1,19 @@
+name: AI CodeQL Fix
+
+on:
+  schedule:
+    - cron: '15 12 * * *'
+
+jobs:
+  call_codeql_reusable_workflow:
+    uses: AMD-GH-Actions/ai-pr-platform-actions-lib/.github/workflows/reusable_codeql.yml@main
+    with:
+      team_name: rocm_clr
+      alerts: ''
+      from_date: ''
+      to_date: ''
+      filter_by_rules: ''
+      trigger_event: ${{ github.event_name }}
+    secrets:
+      gh_token: ${{ secrets.AI_GH_TOKEN }}
+      codeql_token: ${{ secrets.AI_CODEQL_API_KEY }}

projects/clr/.github/workflows/ai_codeql_historical.yml

@@ -0,0 +1,36 @@
+name: AI CodeQL Fix - historical
+
+on:
+  workflow_dispatch:
+    inputs:
+      alerts:
+        required: false
+        type: string
+        description: 'ℹ️Use either alert IDs or Start/End date!⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀Alert IDs (comma-separated values and/or list e.g. 1152, 1122-1124)'
+      from_date:
+        required: false
+        type: string
+        description: 'Start date (use YYYY-MM-DD format)'
+      to_date:
+        required: false
+        type: string
+        description: 'End date (use YYYY-MM-DD format) ⠀⠀⠀ ⚠️Must be within 15 days of Start date!'
+      filter_by_rules:
+        required: false
+        type: string
+        description: 'Filter by Rule IDs ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ ⚠️Overrides team-configured rules⠀⠀⠀⠀ https://amd.atlassian.net/wiki/x/tSrkNg'
+
+jobs:
+  call_codeql_reusable_workflow:
+    uses: AMD-GH-Actions/ai-pr-platform-actions-lib/.github/workflows/reusable_codeql.yml@main
+    with:
+      team_name: rocm_clr
+      alerts: ${{ inputs.alerts }}
+      from_date: ${{ inputs.from_date }}
+      to_date: ${{ inputs.to_date }}
+      filter_by_rules: ${{ inputs.filter_by_rules }}
+      trigger_event: ${{ github.event_name }}
+    secrets:
+      gh_token: ${{ secrets.AI_GH_TOKEN }}
+      codeql_token: ${{ secrets.AI_CODEQL_API_KEY }}
+ 

projects/clr/.github/workflows/clang-format.yml

@@ -0,0 +1,22 @@
+name: Clang format check  
+on:
+  pull_request:
+    types: [synchronize, opened]
+
+jobs:
+  format:
+    runs-on: AMD-ROCm-Internal-dev1
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install clang-format
+        run: |
+          sudo apt update && sudo apt install -y clang-format
+
+      - name: Run clang-format-check
+        id: clang-format
+        run: |
+          chmod +x .github/hooks/clang-format-check.sh
+          ./.github/hooks/clang-format-check.sh --range "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"

projects/clr/.github/workflows/keyword-check.yml

@@ -0,0 +1,73 @@
+name: Keywords checker
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
+    branches:
+      - amd-staging
+  workflow_dispatch:
+
+jobs:
+  check-keywords:
+    runs-on: AMD-ROCm-Internal-dev1
+    env:
+      KEYWORDS: ${{ vars.KEYWORDS }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Check keywords
+        run: |
+          set -e
+
+          if [ -z "$KEYWORDS" ]; then
+            echo "No keywords set. Skipping check"
+            exit 0
+          fi
+
+          IFS=',' read -ra KEYWORDS_ARRAY <<< "$KEYWORDS"
+          echo "Checking against list of keywords: ${KEYWORDS_ARRAY[*]}"
+
+          MATCHED=0
+          BASE_BRANCH=${{github.event.pull_request.base.ref}}
+          HEAD_BRANCH=${{github.event.pull_request.head.ref}}
+          PR_TITLE="${{ github.event.pull_request.title }}"
+
+          for file in $(git diff --name-only origin/$BASE_BRANCH..origin/$HEAD_BRANCH); do
+            if [ -f "$file" ]; then
+              for keyword in "${KEYWORDS_ARRAY[*]}"; do
+                grep -in -E "${keyword}" "$file" | while IFS= read -r line; do
+                  echo "Matched in '$file': $line"
+                  MATCHED=1
+                done
+              done
+            fi
+          done
+
+          for commit in $(git log --format=%H origin/$BASE_BRANCH..origin/$HEAD_BRANCH); do
+            msg=$(git log -1 --format=%B "$commit")
+            for keyword in "${KEYWORDS_ARRAY[*]}"; do
+              if echo "$msg" | grep -i -q "$keyword"; then
+                echo "Match in commit $commit: $msg"
+                MATCHED=1
+              fi
+            done
+          done
+
+          for keyword in "${KEYWORDS_ARRAY[*]}"; do
+            if echo "$PR_TITLE" | grep -i -q "$keyword"; then
+              echo "Match in PR title"
+              MATCHED=1
+            fi
+          done
+          
+          if [ "$MATCHED" -eq 1 ]; then
+            echo "Keywords found, please see diagnostics higher"
+            exit 1
+          else
+            echo "No keywords found"
+            exit 0
+          fi

projects/clr/.github/workflows/kws-caller.yml

@@ -0,0 +1,15 @@
+name: Rocm Validation Suite KWS
+on:
+  push:
+    branches: [amd-staging, amd-mainline]
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+jobs:
+  kws:
+    if: ${{ github.event_name == 'pull_request' }}
+    uses: AMD-ROCm-Internal/rocm_ci_infra/.github/workflows/kws.yml@mainline
+    secrets: inherit
+    with:
+      pr_number: ${{github.event.pull_request.number}}
+      base_branch: ${{github.base_ref}}

projects/clr/.github/workflows/pr-title-validate.yml

@@ -0,0 +1,46 @@
+name: Validate PR Title
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+
+jobs:
+  validate-pr-title:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check PR Title
+        id: check-pr-title
+        run: |
+          PR_TITLE="${{ github.event.pull_request.title }}"
+
+          if [[ ! "$PR_TITLE" =~ ^SWDEV-[0-9]+ ]]; then
+            echo "::error::PR title must start with a Jira ticket ID, SWDEV-<num>"
+            exit 1
+          else
+            echo "PR title is valid"
+          fi
+
+  validate-commit-messages:
+    runs-on: AMD-ROCm-Internal-dev1
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      - name: Check all commit messages
+        id: validate-commit-messags
+        run: |
+          COMMITS=$(git log --format="%H %s" origin/${{ github.event.pull_request.base.ref }}..origin/${{ github.event.pull_request.head.ref }})
+          echo "$COMMITS"
+          echo "$COMMITS" | while read -r hash message; do
+            echo -e "$hash $message\n "
+            if [[ "$message" =~ ^SWDEV-[0-9]+ ]]; then
+              echo "Valid JIRA ticket format"
+            elif [[ "$message" =~ ^Merge\ branch ]]; then
+              echo "Merge commits are allowed"
+            else
+              echo "::error:: $hash commit should start with Jira ticket ID, SWDEV-<num> or be a merge commit"
+              exit 1
+            fi
+          done

projects/clr/.github/workflows/rocm-ci-caller.yml

@@ -0,0 +1,25 @@
+name: ROCm CI Caller  
+on:  
+  pull_request:  
+    branches: [amd-staging, amd-npi, release/rocm-rel-*, amd-mainline]  
+    types: [opened, reopened, synchronize]  
+  push:  
+    branches: [amd-mainline]  
+  workflow_dispatch:  
+  issue_comment:  
+    types: [created]  
+  
+jobs:  
+  call-workflow:  
+    if: github.event_name != 'issue_comment' ||(github.event_name == 'issue_comment' && github.event.issue.pull_request && (startsWith(github.event.comment.body, '!verify') || startsWith(github.event.comment.body, '!linux-hip-psdb') || startsWith(github.event.comment.body, '!verify release') || startsWith(github.event.comment.body, '!verify retest')))  
+    uses: AMD-ROCm-Internal/rocm_ci_infra/.github/workflows/rocm_ci.yml@mainline
+    secrets: inherit  
+    with:  
+      input_sha: ${{github.event_name == 'pull_request' && github.event.pull_request.head.sha || (github.event_name == 'push' && github.sha) || (github.event_name == 'issue_comment' && github.event.issue.pull_request.head.sha) || github.sha}}  
+      input_pr_num: ${{github.event_name == 'pull_request' && github.event.pull_request.number || (github.event_name == 'issue_comment' && github.event.issue.number) || 0}}  
+      input_pr_url: ${{github.event_name == 'pull_request' && github.event.pull_request.html_url || (github.event_name == 'issue_comment' && github.event.issue.pull_request.html_url) || ''}}  
+      input_pr_title: ${{github.event_name == 'pull_request' && github.event.pull_request.title || (github.event_name == 'issue_comment' && github.event.issue.pull_request.title) || ''}}  
+      repository_name: ${{ github.repository }}  
+      base_ref: ${{github.event_name == 'pull_request' && github.event.pull_request.base.ref || (github.event_name == 'issue_comment' && github.event.issue.pull_request.base.ref) || github.ref}}  
+      trigger_event_type: ${{ github.event_name }}  
+      comment_text: ${{ github.event_name == 'issue_comment' && github.event.comment.body || '' }}

projects/clr/.github/workflows/validate-pr-description.yml

@@ -0,0 +1,22 @@
+name: Validate PR desription
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize]
+
+jobs:
+  validate-pr-description:
+    runs-on: AMD-ROCm-Internal-dev1
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Validate PR description
+        env:
+          PR_DESCRIPTION: ${{ github.event.pull_request.body }}
+        run: python .github/scripts/validate_pr_description.py

projects/clr/.gitignore

@@ -0,0 +1,46 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Directories
+build/
+release/
+debug/
+packages/
+install/
+.vs/
+.vscode/
+.cache/
+
+# Editor temp files
+*.swp
+*.swo

projects/clr/CHANGELOG.md

@@ -0,0 +1,533 @@
+# Change Log for HIP
+
+Full documentation for HIP is available at [rocm.docs.amd.com](https://rocm.docs.amd.com/projects/HIP/en/latest/index.html)
+
+## HIP 7.0 for ROCm 7.0
+
+### Added
+
+* New HIP APIs
+    - `hipLaunchKernelEx`  dispatches the provided kernel with the given launch configuration and forwards the kernel arguments.
+    - `hipLaunchKernelExC`  launches a HIP kernel using a generic function pointer and the specified configuration.
+    - `hipDrvLaunchKernelEx`  dispatches the device kernel represented by a HIP function object.
+    - `hipMemGetHandleForAddressRange`  gets a handle for the address range requested.
+    - `num_threads`  Total number of threads in the group. The legacy API size is alias.
+* New support for Open Compute Project (OCP) floating-point `FP4`/`FP6`/`FP8` as the following. For details, see [Low precision floating point document](https://rocm.docs.amd.com/projects/HIP/en/latest/reference/low_fp_types.html).
+    - Data types for `FP4`/`FP6`/`FP8`.
+    - HIP APIs for `FP4`/`FP6`/`FP8`, which are compatible with corresponding CUDA APIs.
+    - HIP Extensions APIs for microscaling formats, which are supported on AMD GPUs.
+* New `wptr` and `rptr` values in `ClPrint`, for better logging in dispatch barrier methods.
+* New debug mask, to print precise code object information for logging.
+* The `_sync()` version of crosslane builtins such as `shfl_sync()` and `__reduce_add_sync` are enabled by default. These can be disabled by setting the preprocessor macro `HIP_DISABLE_WARP_SYNC_BUILTINS`.
+
+### Changed
+
+* Some unsupported GPUs such as gfx9, gfx8 and gfx7 are deprecated on Microsoft Windows.
+* Stream validation in some HIP APIs are removed, to match the behavior with CUDA.
+
+### Optimized
+
+HIP runtime has the following functional improvements which greatly improve runtime performance and user experience.
+
+* Reduced usage of the lock scope in events and kernel handling.
+    - Switches to `shared_mutex` for event validation, uses `std::unique_lock` in HIP runtime to create/destroy event, instead of `scopedLock`.
+    - Reduces the `scopedLock` in handling of kernel execution. HIP runtime now calls `scopedLock` during kernel binary creation/initialization,
+    doesn't call it again during kernel vector iteration before launch.
+* Implementation of unifying managed buffer and kernel argument buffer so HIP runtime doesn't need to create/load a separate kernel argument buffer.
+* Refactored memory validation, creates a unique function to validate a variety of memory copy operations.
+* Improved kernel logging using demangling shader names.
+* Advanced support for SPIRV, now kernel compilation caching is enabled by default. This feature is controlled by the environment variable `AMD_COMGR_CACHE`, for details, see [hip_rtc document](https://rocm.docs.amd.com/projects/HIP/en/latest/how-to/hip_rtc.html).
+* Programmatic support for scratch limit on GPU device. Developer can now use the environment variable `HSA_SCRATCH_SINGLE_LIMIT` to change the default allocation size with expected scratch limit.
+* HIP runtime now enables peer-to-peer (P2P) memory copies to utilize all available SDMA engines, rather than being limited to a single engine. It also selects the best engine first to give optimal bindwidth.
+* Improved launch latency for `D2D` copies and `memset` on MI300 series.
+
+### Resolved issues
+
+* Error of "unable to find modules" in HIP clean up for code object module.
+
+## HIP 6.4.2 for ROCm 6.4.2
+
+### Added
+
+* Support for the pointer attribute `HIP_POINTER_ATTRIBUTE_CONTEXT`.
+
+### Optimized
+
+* Improved implementation in `hipEventSynchronize`, HIP runtime now makes internal callbacks non-blocking to gain performance.
+
+### Resolved issues
+
+* Issue of dependency on `libgcc-s1` during rocm-dev install on Debian Buster. HIP runtime removed this Debian package dependency, and uses `libgcc1` instead for this distros.
+* Building issue for `COMGR` dynamic load on Fedora and other Distros. HIP runtime now doesn't link against `libamd_comgr.so`.
+* Failure in the API `hipStreamDestroy`, when stream type is `hipStreamLegacy`. The API now returns error code `hipErrorInvalidResourceHandle` on this condition.
+* Kernel launch errors, such as `shared object initialization failed`, `invalid device function` or `kernel execution failure`. HIP runtime now loads `COMGR` properly considering the file with its name and mapped mage.
+* Memory access fault in some appplications. HIP runtime fixed offset accumulation in memory address.
+
+## HIP 6.4.1 for ROCm 6.4.1
+
+### Added
+
+* New log mask enumeration `LOG_COMGR` enables logging precise code object information.
+
+### Changed
+
+* HIP runtime uses device bitcode before SPIRV.
+* The implementation of preventing `hipLaunchKernel` latency degradation with number of idle streams is reverted/disabled by default.
+
+### Optimized
+
+* Improved kernel logging includes de-mangling shader names.
+* Refined implementation in HIP APIs `hipEventRecords` and `hipStreamWaitEvent` for performance improvement.
+
+### Resolved issues
+
+* Stale state during the graph capture. The return error was fixed, HIP runtime now always uses the latest dependent nodes during `hipEventRecord` capture.
+* Segmentation fault during kernel execution. HIP runtime now allows maximum stack size as per ISA on the GPU device.
+
+## HIP 6.4 (For ROCm 6.4)
+
+### Added
+
+* New HIP APIs
+    - `hipDeviceGetTexture1DLinearMaxWidth`  returns the maximum width of elements in a 1D linear texture, that can be allocated on the specified device.
+    - `hipStreamBatchMemOp`  enqueues an array of batch memory operations in the stream, for stream synchronization.
+    - `hipGraphAddBatchMemOpNode`  creates a batch memory operation node and adds it to a graph.
+    - `hipGraphBatchMemOpNodeGetParams`  returns the pointer of parameters from the batch memory operation node.
+    - `hipGraphBatchMemOpNodeSetParams`  sets parameters for the batch memory operation node.
+    - `hipGraphExecBatchMemOpNodeSetParams`  sets the parameters for a batch memory operation node in the given executable graph.
+    - `hipLinkAddData` adds SPIRV code object data to linker instance with options.
+    - `hipLinkAddFile` adds SPIRV code object file to linker instance with options.
+    - `hipLinkCreate`  creates linker instance at runtime with options.
+    - `hipLinkComplete` completes linking of program and output linker binary to use with hipModuleLoadData.
+    - `hipLinkDestroy`  deletes linker instance.
+
+### Changed
+
+* roc-obj* tools are being deprecated, and will be removed in an upcoming release.
+    - Perl package dependencies are now RECOMMENDS or SUGGESTS.  Users will need to install these themselves.
+    - Support for ROCm Object tooling has moved into llvm-objdump provided by package rocm-llvm.
+* SDMA retainer logic is removed for engine selection in operation of runtime buffer copy.
+
+### Optimized
+
+* `hipGraphLaunch` parallelism is improved for complex data-parallel graphs.
+* Round-robin queue mechanism is updated for command scheduling. For multi-streams execution, HSA queue from null stream lock is freed and won't occupy the queue ID after the kernel in the stream is finished.
+* The HIP runtime doesn't free bitcode object before code generation. It adds a cache, which allows compiled code objects to be reused instead of recompiling. This improves performance on multi-GPU systems.
+* Runtime uses unified copy approach
+    - Unpinned `H2D`copies are no longer blocking until the size of 1MB.
+    - Kernel copy path is enabled for unpinned `H2D`/`D2H` methods.
+    - The default environment variable `GPU_FORCE_BLIT_COPY_SIZE` is set to `16`, which limits the kernel copy to sizes less than 16 KB, while copies about that would be handled by `SDMA` engine.
+    - Blit code is refactored and ASAN instrumentation is cleaned up.
+* HIP runtime uses signals without interrupts.
+    - In active wait mode, uses signals without interrupts by default.
+    - Only when a callback is required, switches to the interrupts.
+
+### Resolved issues
+
+* Out of memory error on Windows. When the user calls `hipMalloc` for device memory allocation while specifying a size larger than the available device memory, the HIP runtime fixes the error in the API implementation, allocating the available device memory plus system memory (shared virtual memory).
+* Error of dependency on libgcc-s1 during rocm-dev install on Debian Buster. HIP runtime now uses libgcc1 for this distros.
+* Stack corruption during kernel execution. HIP runtime now adds maximum stack size limit based on the GPU device feature.
+
+### Upcoming changes
+
+The following are the list of backwards incompatible changes planned for the upcoming major ROCm release.
+
+* Signature changes in APIs to match corresponding CUDA APIs,
+    - `hiprtcCreateProgram`
+    - `hiprtcCompileProgram`
+    - `hipCtxGetApiVersion`
+* Behavior of `hipPointerGetAttributes` is changed to match corresponding CUDA API in version 11 and later releases.
+* Behavior of `hipFree` is changed to match corresponding CUDA API `cudaFree`.
+* HIP vector constructor changes for `hipComplex`.
+* Return error/value codes update in the following hip APIs, they now match the corresponding CUDA APIs,
+    - `hipModuleLaunchKernel`
+    - `hipExtModuleLaunchKernel`
+    - `hipModuleLaunchCooperativeKernel`
+    - `hipGetTextureAlignmentOffset`
+    - `hipTexObjectCreate`
+    - `hipBindTexture2D`
+    - `hipBindTextureToArray`
+    - `hipModuleLoad`
+    - `hipLaunchCooperativeKernelMultiDevice`
+    - `hipExtLaunchCooperativeKernelMultiDevice`
+ 
+* HIPRTC implementation, the compilation of hiprtc now uses  namespace ` __hip_internal`, instead of the standard headers `std`.
+* Stream capture mode update in the following hip APIs. Stream can only be captured in relax mode, to match the behavior of the corresponding CUDA APIs,
+   - `hipMallocManaged`
+   - `hipMemAdvise`
+   - `hipLaunchCooperativeKernelMultiDevice`
+   - `hipDeviceSetCacheConfig`
+   - `hipDeviceSetSharedMemConfig`
+   - `hipMemPoolCreate`
+   - `hipMemPoolDestory`
+   - `hipDeviceSetMemPool`
+   - `hipEventQuery`
+* The implementation of `hipStreamAddCallback` is updated, to match the behavior of CUDA.
+* Removal of hiprtc symbols from hip library.
+    - hiprtc will be a independent library, all symbols supported in hip library are removed.
+    - Any application using hiprtc APIs should link explicitly with hiprtc library.
+    - This change makes the usage of hiprtc library on Linux the same as on Windows, and matches the behavior of CUDA nvrtc.
+* Removal of deprecated struct `HIP_MEMSET_NODE_PARAMS`, developers can use definition `hipMemsetParams` instead.
+
+
+## HIP 6.3.2 for ROCm 6.3.2
+
+### Added
+
+* Tracking of Heterogeneous System Architecture (HSA) handlers:
+    - Adds an atomic counter to track the outstanding HSA handlers.
+    - Waits on CPU for the callbacks if the number exceeds the defined value.
+* Codes to capture Architected Queueing Language (AQL) packets for HIP graph memory copy node between host and device. HIP enqueues AQL packets during graph launch.
+* Control to use system pool implementation in runtime commands handling. By default, it is disabled.
+* A new path to avoid `WaitAny` calls in `AsyncEventsLoop`. The new path is selected by default.
+* Runtime control on decrement counter only if event is popped. There is a new way to restore dead signals cleanup for the old path.
+* A new logic in runtime to track the age of events from the kernel mode driver.
+
+### Optimized
+
+* HSA callback performance. The HIP runtime creates and submits commands in the queue and interacts with HSA through a callback function. HIP waits for the CPU status from HSA to optimize handling of events, profiling, commands, and HSA signals for higher performance.
+* Runtime optimisation which combines all logic of `WaitAny` in a single processing loop and avoids extra memory allocations or reference counting. The runtime won't spin on the CPU if all events are busy.
+* Multi-threaded dispatches for performance improvement.
+* Command submissions and processing between CPU and GPU by introducing a way to limit the software batch size.
+* Switch to `std::shared_mutex` in book/keep logic in streams from multiple threads simultaneously, for performance improvement in specific customer applications.
+* `std::shared_mutex` is used in memory object mapping, for performance improvement.
+
+### Resolved issues
+
+* Race condition in multi-threaded producer/consumer scenario with `hipMallocFromPoolAsync`.
+* Segmentation fault with `hipStreamLegacy` while using the API `hipStreamWaitEvent`.
+* Usage of `hipStreamLegacy` in HIP event record.
+* A soft hang in graph execution process from HIP user object. The fix handles the release of graph execution object properly considering synchronization on the device/stream. The user application now behaves the same with  hipUserObject  on both the AMD ROCm and NVIDIA CUDA platforms.
+
+
+## HIP 6.3.1 for ROCm 6.3.1
+
+### Added
+
+* An activeQueues set that tracks only the queues that have a command submitted to them, which allows fast iteration in `waitActiveStreams`.
+
+### Optimized
+
+* Mechanism of preventing `hipLaunchKernel` latency degradation with number of idle streams is implemented for performance improvement.
+
+## HIP 6.3 for ROCm 6.3
+
+### Added
+
+* New HIP APIs
+    - `hipGraphExecGetFlags`  returns the flags on executable graph.
+    - `hipGraphNodeSetParams`  updates parameters of a created node.
+    - `hipGraphExecNodeSetParams`  updates parameters of a created node on executable graph.
+    - `hipDrvGraphMemcpyNodeGetParams`  gets a memcpy node's parameters.
+    - `hipDrvGraphMemcpyNodeSetParams`  sets a memcpy node's parameters.
+    - `hipDrvGraphAddMemFreeNode`  creates a memory free node and adds it to a graph.
+    - `hipDrvGraphExecMemcpyNodeSetParams`  sets the parameters for a memcpy node in the given graphExec.
+    - `hipDrvGraphExecMemsetNodeSetParams`  sets the parameters for a memset node in the given graphExec.
+
+### Changed
+
+* Un-deprecated HIP APIs
+    - `hipHostAlloc`
+    - `hipFreeHost`
+
+### Optimized
+
+* Disabled CPU wait in device synchronize to avoid idle time in applications such as Hugging Face models and PyTorch.
+* Optimized multi-threaded dispatches to improve performance.
+* Limited the software batch size to control the number of command submissions for runtime to handle efficiently.
+* Optimizes HSA callback performance when a large number of events are recorded by multiple threads and submitted to multiple GPUs.
+* HIP graph execution perfomance improvement.
+    - Added the optimized multistream path in graph execution. It uses a fixed number of async streams in the execution
+    - Optimized the launch latency, where commands creation and execution is done at the same time
+    - Optimized the scheduling to use less barriers and waiting signals if the same queue  can be detected
+    - The new path is controlled by a new environment variable, with the options either to use the original path, or to force the number of asynchronous queues for execution.
+
+### Resolved issues
+
+* Soft hang in runtime wait event when run TensorFlow.
+* Memory leak in the API `hipGraphInstantiate` when kernel is launched using `hipExtLaunchKernelGGL` with event.
+* Memory leak when the API `hipGraphAddMemAllocNode` is called.
+* The `_sync()` version of crosslane builtins such as `shfl_sync()`,
+  `__all_sync()` and `__any_sync()`, continue to be hidden behind the
+  preprocessor macro `HIP_ENABLE_WARP_SYNC_BUILTINS`, and will be enabled
+  unconditionally in the next ROCm release.
+
+
+## HIP 6.2.41134 for ROCm 6.2.1
+
+### Resolved issues
+
+* Soft hang when use AMD_SERIALIZE_KERNEL.
+* Memory leak in hipIpcCloseMemHandle.
+
+
+## HIP 6.2 (For ROCm 6.2)
+
+### Added
+- Introduced the `_sync()` version of crosslane builtins such as `shfl_sync()`, `__all_sync()`
+  and `__any_sync()`. These take a 64-bit integer as an explicit mask argument.
+  - In HIP 6.2, these are hidden behind the preprocessor macro
+    `HIP_ENABLE_WARP_SYNC_BUILTINS`, and will be enabled unconditionally in HIP 6.3.
+- Added new HIP APIs
+    - `hipGetProcAddress` returns the pointer to driver function, corresponding to the defined driver function symbol.
+    - `hipGetFuncBySymbol` returns the pointer to device entry function that matches entry function symbolPtr.
+    - `hipStreamBeginCaptureToGraph` begins graph capture on a stream to an existing graph.
+    - `hipGraphInstantiateWithParams`  creates an executable graph from a graph.
+    - `hipMemcpyAtoA`  copies from one 1D array to another.
+    - `hipMemcpyDtoA`  copies from device memory to a 1D array.
+    - `hipMemcpyAtoD`  copies from one 1D array to device memory.
+    - `hipMemcpyAtoHAsync`  copies from one 1D array to host memory.
+    - `hipMemcpyHtoAAsync`  copies from host memory to a 1D array.
+    - `hipMemcpy2DArrayToArray`  copies data between host and device.
+
+- Added a new flag `integrated` support in device property
+
+    The `integrated` flag is added in the struct `hipDeviceProp_t`.
+    On the integrated `APU` system, the runtime driver detects and sets this flag to `1`, in which case the API `hipDeviceGetAttribute` returns enum `hipDeviceAttribute_t` for hipDeviceAttributeIntegrated as value `1`, for integrated GPU device.
+
+    The enum value `hipDeviceAttributeIntegrated` corresponds to `cudaDevAttrIntegrated` on CUDA platform.
+- Added initial support for 8-bit floating point datatype in `amd_hip_fp8.h`. These are accessible via `#include <hip/hip_fp8.h>`
+- Add UUID support for environment variable `HIP_VISIBLE_DEVICES`.
+
+### Resolved issues
+- Stream capture support in HIP graph.
+Prohibited and unhandled operations are fixed during stream capture in HIP runtime.
+- Fix undefined symbol error for hipTexRefGetArray & hipTexRefGetBorderColor.
+
+## HIP 6.1 (For ROCm 6.1)
+
+### Added
+- New environment variable HIP_LAUNCH_BLOCKING
+It is used for serialization on kernel execution.
+The default value is 0 (disable), kernel will execute normally as defined in the queue. When this environment variable is set as 1 (enable), HIP runtime will serialize kernel enqueue, behaves the same as AMD_SERIALIZE_KERNEL.
+- Added HIPRTC support for hip headers driver_types, math_functions, library_types, math_functions, hip_math_constants, channel_descriptor, device_functions, hip_complex, surface_types, texture_types.
+
+### Changed
+- HIPRTC now assumes WGP mode for gfx10+. CU mode can be enabled by passing `-mcumode` to the compile options from `hiprtcCompileProgram`.
+
+### Resolved issues
+- HIP complex vector type multiplication and division operations.
+On AMD platform, some duplicated complex operators are removed to avoid compilation failures.
+In HIP, hipFloatComplex and hipDoubleComplex are defined as complex data types,
+typedef float2 hipFloatComplex;
+typedef double2 hipDoubleComplex;
+Any application uses complex multiplication and division operations, need to replace '*' and '/' operators with the following,
+    - hipCmulf() and hipCdivf() for hipFloatComplex
+    - hipCmul() and hipCdiv() for hipDoubleComplex
+
+    Note: These complex operations are equivalent to corresponding types/functions on NVIDIA platform.
+
+## HIP 6.0 (For ROCm 6.0)
+
+### Added
+- Addition of hipExtGetLastError
+  - AMD backend specific API, to return error code from last HIP API called from the active host thread
+
+- New fields for external resource interoperability,
+  - Structs
+    - hipExternalMemoryHandleDesc_st
+    - hipExternalMemoryBufferDesc_st
+    - hipExternalSemaphoreHandleDesc_st
+    - hipExternalSemaphoreSignalParams_st
+    - hipExternalSemaphoreWaitParams_st
+  - Enumerations
+    - hipExternalMemoryHandleType_enum
+    - hipExternalSemaphoreHandleType_enum
+    - hipExternalMemoryHandleType_enum
+
+- New members are added in HIP struct hipDeviceProp_t, for new feature capabilities including,
+  - Texture
+     - int maxTexture1DMipmap;
+     - int maxTexture2DMipmap[2];
+     - int maxTexture2DLinear[3];
+     - int maxTexture2DGather[2];
+     - int maxTexture3DAlt[3];
+     - int maxTextureCubemap;
+     - int maxTexture1DLayered[2];
+     - int maxTexture2DLayered[3];
+     - int maxTextureCubemapLayered[2];
+  - Surface
+     - int maxSurface1D;
+     - int maxSurface2D[2];
+     - int maxSurface3D[3];
+     - int maxSurface1DLayered[2];
+     - int maxSurface2DLayered[3];
+     - int maxSurfaceCubemap;
+     - int maxSurfaceCubemapLayered[2];
+  - Device
+     - hipUUID uuid;
+     - char luid[8];
+       -- this is 8-byte unique identifier. Only valid on windows
+       -- LUID (Locally Unique Identifier) is supported for interoperability between devices.
+     - unsigned int luidDeviceNodeMask; \
+
+     Note: HIP supports LUID only on Windows OS.
+- Added `amd_hip_bf16.h` which adds `bfloat16` type. These definitions are accessible via `#include <hip/hip_bf16.h>`
+This header exists alongside the older bfloat16 header in`amd_hip_bfloat16.h` which is included via `hip/hip_bfloat16.h`. Users are recommended to use `<hip/hip_bf16.h>` instead of `<hip/hip_bfloat16.h>`.
+
+### Changed
+- Some OpenGL Interop HIP APIs are moved from the hip_runtime_api header to a new header file hip_gl_interop.h for the AMD platform, as following,
+    - hipGLGetDevices
+    - hipGraphicsGLRegisterBuffer
+    - hipGraphicsGLRegisterImage
+- With ROCm 6.0, the HIP version is 6.0. As the HIP runtime binary suffix is updated in every major ROCm release, in ROCm 6.0, the new filename is libamdhip64.so.6. Furthermore, in ROCm 6.0 release, the libamdhip64.so.5 binary from ROCm 5.7 is made available to maintain binary backward compatibility with ROCm 5.x.
+
+### Changed Impacting Backward Compatibility
+- Data types for members in HIP_MEMCPY3D structure are changed from "unsigned int" to "size_t".
+- The value of the flag hipIpcMemLazyEnablePeerAccess is changed to “0x01”, which was previously defined as “0”.
+- Some device property attributes are not currently support in HIP runtime, in order to maintain consistency, the following related enumeration names are changed in hipDeviceAttribute_t
+    - hipDeviceAttributeName is changed to hipDeviceAttributeUnused1
+    - hipDeviceAttributeUuid is changed to hipDeviceAttributeUnused2
+    - hipDeviceAttributeArch is changed to hipDeviceAttributeUnused3
+    - hipDeviceAttributeGcnArch is changed to hipDeviceAttributeUnused4
+    - hipDeviceAttributeGcnArchName is changed to hipDeviceAttributeUnused5
+- HIP struct hipArray is removed from driver type header to be complying with cuda
+- hipArray_t replaces hipArray*, as the pointer to array.
+    - This allows hipMemcpyAtoH and hipMemcpyHtoA to have the correct array type which is equivalent to coresponding CUDA driver APIs.
+
+### Removed
+- Deprecated Heterogeneous Compute (HCC) symbols and flags are removed from the HIP source code, including,
+    - Build options on obsolete HCC_OPTIONS was removed from cmake.
+    - Micro definitions are removed.
+      HIP_INCLUDE_HIP_HCC_DETAIL_DRIVER_TYPES_H
+      HIP_INCLUDE_HIP_HCC_DETAIL_HOST_DEFINES_H
+    - Compilation flags for the platform definitions,
+      AMD platform,
+      __HIP_PLATFORM_HCC__
+      __HCC__
+      __HIP_ROCclr__
+      NVIDIA platform,
+      __HIP_PLATFORM_NVCC__
+- File directories in the clr repository are removed,
+  https://github.com/ROCm/clr/blob/develop/hipamd/include/hip/hcc_detail
+  https://github.com/ROCm/clr/blob/develop/hipamd/include/hip/nvcc_detail
+- Deprecated gcnArch is removed from hip device struct hipDeviceProp_t.
+- Deprecated "enum hipMemoryType memoryType;" is removed from HIP struct hipPointerAttribute_t union.
+- Deprecated HIT based tests are removed from HIP project
+- Catch tests are available [hip-tests] (https://github.com/ROCm/hip-tests) project
+
+### Resolved issues
+- Kernel launch maximum dimension validation is added specifically on gridY and gridZ in the HIP API hipModule-LaunchKernel. As a result,when hipGetDeviceAttribute is called for the value of hipDeviceAttributeMaxGrid-Dim, the behavior on the AMD platform is equivalent to NVIDIA.
+- The HIP stream synchronisation behavior is changed in internal stream functions, in which a flag "wait" is added and set when the current stream is null pointer while executing stream synchronisation on other explicitly created streams. This change avoids blocking of execution on null/default stream.
+The change won't affect usage of applications, and makes them behave the same on the AMD platform as NVIDIA.
+- Error handling behavior on unsupported GPU is fixed, HIP runtime will log out error message, instead of creating signal abortion error which is invisible to developers but continued kernel execution process. This is for the case when developers compile any application via hipcc, setting the option --offload-arch with GPU ID which is different from the one on the system.
+
+### Known Issues
+- Dynamically loaded HIP runtime library references incorrect version of hipDeviceGetProperties and hipChooseDevice APIs
+
+When an application dynamically loads the HIP runtime library from ROCm 6.0 and attempts to get the hipDeviceGetProperties and/or hipChooseDevice entry-points using dlsym, the application gets the older version (ROCm 5.7) of those entry-points.
+
+As a workaround, while compiling with ROCm 6.0, use the string "hipDeviceGetPropertiesR0600", and "hipChooseDeviceR0600" respectively for hipDeviceGetProperties and hipChooseDevice APIs.
+
+## HIP 5.7.1 (For ROCm 5.7.1)
+
+### Resolved issues
+- hipPointerGetAttributes API returns the correct HIP memory type as hipMemoryTypeManaged for managed memory.
+
+## HIP 5.7 (For ROCm 5.7)
+
+### Added
+- Added meta_group_size/rank for getting the number of tiles and rank of a tile in the partition
+- Added new APIs supporting Windows only, under development on Linux
+
+    - hipMallocMipmappedArray for allocating a mipmapped array on the device
+
+    - hipFreeMipmappedArray for freeing a mipmapped array on the device
+
+    - hipGetMipmappedArrayLevel for getting a mipmap level of a HIP mipmapped array
+
+    - hipMipmappedArrayCreate for creating a mipmapped array
+
+    - hipMipmappedArrayDestroy for destroy a mipmapped array
+
+    - hipMipmappedArrayGetLevel for getting a mipmapped array on a mipmapped level
+
+### Known Issues
+- HIP memory type enum values currently don't support equivalent value to cudaMemoryTypeUnregistered, due to HIP functionality backward compatibility.
+- HIP API hipPointerGetAttributes could return invalid value in case the input memory pointer was not allocated through any HIP API on device or host.
+
+### Upcoming changes
+- Removal of gcnarch from hipDeviceProp_t structure
+- Addition of new fields in hipDeviceProp_t structure
+  - maxTexture1D
+  - maxTexture2D
+  - maxTexture1DLayered
+  - maxTexture2DLayered
+  - sharedMemPerMultiprocessor
+  - deviceOverlap
+  - asyncEngineCount
+  - surfaceAlignment
+  - unifiedAddressing
+  - computePreemptionSupported
+  - hostRegisterSupported
+  - uuid
+- Removal of deprecated code
+  -hip-hcc codes from hip code tree
+- Correct hipArray usage in HIP APIs such as hipMemcpyAtoH and hipMemcpyHtoA
+- HIPMEMCPY_3D fields correction to avoid truncation of "size_t" to "unsigned int" inside hipMemcpy3D()
+- Renaming of 'memoryType' in hipPointerAttribute_t structure to 'type'
+- Correct hipGetLastError to return the last error instead of last API call's return code
+- Update hipExternalSemaphoreHandleDesc to add "unsigned int reserved[16]"
+- Correct handling of flag values in hipIpcOpenMemHandle for hipIpcMemLazyEnablePeerAccess
+- Remove hiparray* and make it opaque with hipArray_t
+
+## HIP 5.6.1 (For ROCm 5.6.1)
+
+### Resolved issues
+- Enabled xnack+ check in HIP catch2 tests hang while tests execution
+- Memory leak when code object files are loaded/unloaded via hipModuleLoad/hipModuleUnload APIs
+- Resolved an issue of crash while using hipGraphAddMemFreeNode
+
+## HIP 5.6 (For ROCm 5.6)
+
+### Added
+- Added hipRTC support for amd_hip_fp16
+- Added hipStreamGetDevice implementation to get the device assocaited with the stream
+- Added HIP_AD_FORMAT_SIGNED_INT16 in hipArray formats
+- hipArrayGetInfo for getting information about the specified array
+- hipArrayGetDescriptor for getting 1D or 2D array descriptor
+- hipArray3DGetDescriptor to get 3D array descriptor
+
+### Changed
+- hipMallocAsync to return success for zero size allocation to match hipMalloc
+- Separation of hipcc perl binaries from HIP project to hipcc project. hip-devel package depends on newly added hipcc package
+- Consolidation of hipamd, ROCclr, and OpenCL repositories into a single repository called clr. Instructions are updated to build HIP from sources in the HIP Installation guide
+- Removed hipBusBandwidth and hipCommander samples from hip-tests
+
+### Optimized
+- Consolidation of hipamd, rocclr and OpenCL projects in clr
+- Optimized lock for graph global capture mode
+
+### Resolved issues
+- Fixed regression in hipMemCpyParam3D when offset is applied
+
+### Known Issues
+- Limited testing on xnack+ configuration
+  - Multiple HIP tests failures (gpuvm fault or hangs)
+- hipSetDevice and hipSetDeviceFlags APIs return hipErrorInvalidDevice instead of hipErrorNoDevice, on a system without GPU
+- Known memory leak when code object files are loaded/unloaded via hipModuleLoad/hipModuleUnload APIs. Issue will be fixed in future release
+
+### Upcoming changes
+- Removal of gcnarch from hipDeviceProp_t structure
+- Addition of new fields in hipDeviceProp_t structure
+  - maxTexture1D
+  - maxTexture2D
+  - maxTexture1DLayered
+  - maxTexture2DLayered
+  - sharedMemPerMultiprocessor
+  - deviceOverlap
+  - asyncEngineCount
+  - surfaceAlignment
+  - unifiedAddressing
+  - computePreemptionSupported
+  - hostRegisterSupported
+  - uuid
+- Removal of deprecated code
+  -hip-hcc codes from HIP code tree
+- Correct hipArray usage in HIP APIs such as hipMemcpyAtoH and hipMemcpyHtoA
+- HIPMEMCPY_3D fields correction to avoid truncation of "size_t" to "unsigned int" inside hipMemcpy3D()
+- Renaming of 'memoryType' in hipPointerAttribute_t structure to 'type'
+- Correct hipGetLastError to return the last error instead of last API call's return code
+- Update hipExternalSemaphoreHandleDesc to add "unsigned int reserved[16]"
+- Correct handling of flag values in hipIpcOpenMemHandle for hipIpcMemLazyEnablePeerAccess
+- Remove hiparray* and make it opaque with hipArray_t

projects/clr/CMakeLists.txt

@@ -0,0 +1,127 @@
+# Copyright (c) 2022 - 2023 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+cmake_minimum_required(VERSION 3.16.8)
+project(clr)
+
+##########
+# Defaults
+##########
+if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    add_compile_options("/wd4267" "/wd4244" "/wd4996")
+    string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+    string(REPLACE "/W3" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+        add_compile_options(/MTd)
+        add_compile_options(-D_ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH)
+    endif()
+endif()
+
+if (AMD_COMPUTE_WIN)
+    add_compile_options("/MT")
+    set(AMD_LIBELF_PATH "${AMD_COMPUTE_WIN}/hsail-compiler/lib/loaders/elf/utils/libelf")
+    set(AMD_SC_PATH "${AMD_COMPUTE_WIN}/sc")
+    message("Public Compute Windows build path: ${AMD_COMPUTE_WIN}, SC: ${AMD_SC_PATH}, LibElf: ${AMD_LIBELF_PATH}")
+endif()
+
+option(CLR_BUILD_HIP "Build HIP" OFF)
+option(CLR_BUILD_OCL "Build OCL" OFF)
+
+# Set default build type
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE "Release")
+endif()
+
+#############
+# Build steps
+#############
+
+# Attempt to auto-detect HIP_PLATFORM by interrogating hipconfig. This is kept
+# for compatibility: users are advised to pass HIP_PLATFORM explicitly.
+# Sets the HIP_PLATFORM variable in the parent scope.
+function(_hip_clr_auto_detect_hip_platform)
+    if (UNIX)
+      set(HIPCC_EXECUTABLE "hipcc")
+      set(HIPCONFIG_EXECUTABLE "hipconfig")
+    else()
+      set(HIPCC_EXECUTABLE "hipcc.exe")
+      set(HIPCONFIG_EXECUTABLE "hipconfig.exe")
+    endif()
+
+    # Set default HIPCC_BIN_DIR to /opt/rocm/bin
+    if(NOT DEFINED HIPCC_BIN_DIR AND UNIX)
+        set(HIPCC_BIN_DIR "/opt/rocm/bin")
+    endif()
+    message(STATUS "Auto-detect HIP_PLATFORM from HIPCC Binary Directory: ${HIPCC_BIN_DIR}")
+
+    if(NOT EXISTS ${HIPCC_BIN_DIR}/${HIPCONFIG_EXECUTABLE})
+        message(FATAL_ERROR "Please pass hipcc/build or hipcc/bin using -DHIPCC_BIN_DIR.")
+    endif()
+
+    # Determine HIP_PLATFORM
+    if(NOT DEFINED HIP_PLATFORM)
+        if(NOT DEFINED ENV{HIP_PLATFORM})
+            execute_process(COMMAND ${HIPCC_BIN_DIR}/${HIPCONFIG_EXECUTABLE} --platform
+                OUTPUT_VARIABLE _detected_hip_platform
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+            set(HIP_PLATFORM "${_detected_hip_platform}" PARENT_SCOPE)
+        else()
+            set(HIP_PLATFORM "$ENV{HIP_PLATFORM}" PARENT_SCOPE)
+        endif()
+    endif()
+endfunction()
+if(CLR_BUILD_HIP)
+    message(STATUS "HIP Common Directory: ${HIP_COMMON_DIR}")
+    if(NOT DEFINED HIP_COMMON_DIR)
+        message(FATAL_ERROR "Please pass HIP using -DHIP_COMMON_DIR. HIP_COMMON_DIR is incorrect")
+    endif()
+    if(NOT DEFINED HIP_PLATFORM)
+        _hip_clr_auto_detect_hip_platform()
+    endif()
+endif()
+
+if(CLR_BUILD_HIP)
+    option(BUILD_SHARED_LIBS "Build the shared library" ON)
+    if (NOT BUILD_SHARED_LIBS)
+        add_compile_definitions(BUILD_STATIC_LIBS)
+    endif()
+endif()
+if((CLR_BUILD_HIP AND HIP_PLATFORM STREQUAL "amd") OR CLR_BUILD_OCL)
+    add_subdirectory(rocclr)
+elseif(HIP_PLATFORM STREQUAL "amd")
+    message(FATAL_ERROR "Please enable building of one or more of the below runtimes:\n- HIP (-DCLR_BUILD_HIP=ON)\n- OpenCL (-DCLR_BUILD_OCL-ON)")
+endif()
+if(CLR_BUILD_HIP)
+    add_subdirectory(hipamd)
+endif()
+if(CLR_BUILD_OCL)
+    add_subdirectory(opencl)
+endif()
+
+#############################
+# Code formatting
+#############################
+# Target: clangformat
+find_program(CLANGFORMAT_EXE clang-format PATHS "/opt/rocm/lib/llvm/bin")
+if(CLANGFORMAT_EXE)
+    file(GLOB_RECURSE FORMAT_SOURCE_FILE_LIST *.cpp *.hpp *.hh *.h *.cc *.c)
+    add_custom_target(clangformat COMMAND ${CLANGFORMAT_EXE} -style=file -i ${FORMAT_SOURCE_FILE_LIST}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+endif()

projects/clr/CODEOWNERS

@@ -0,0 +1,2 @@
+* @cpaquot_amdeng @gandryey_amdeng @skudchad_amdeng @lmoriche_amdeng @axie_amdeng
+

projects/clr/CONTRIBUTING.md

@@ -0,0 +1,142 @@
+# Contributing to HIP/CLR #
+
+We welcome contributions to the HIP project.
+CLR is a part of HIP runtime for the AMD platform.Please follow these details to help ensure your contributions will be successfully accepted.
+
+## Issue Discussion ##
+
+Please use the [GitHub Issue](https://github.com/ROCm/clr/issues) tab to notify us of issues.
+
+* Use your best judgement for issue creation. If your issue is already listed, upvote the issue and
+  comment or post to provide additional details, such as how you reproduced this issue.
+* If you're not sure if your issue is the same, err on the side of caution and file your issue.
+  You can add a comment to include the issue number (and link) for the similar issue. If we evaluate
+  your issue as being the same as the existing issue, we'll close the duplicate.
+* If your issue doesn't exist, use the issue template to file a new issue.
+  * When filing an issue, be sure to provide as much information as possible, including script output so
+    we can collect information about your configuration. This helps reduce the time required to
+    reproduce your issue.
+  * Check your issue regularly, as we may require additional information to successfully reproduce the
+    issue.
+* You may also open an issue to ask questions to the maintainers about whether a proposed change
+  meets the acceptance criteria, or to discuss an idea pertaining to the library.
+
+## Acceptance Criteria ##
+
+clr Compute Language Runtime contains C++ codes for the implementation of HIP runtime APIs on the AMD platform.
+Bug fixes and performance are both important goals in clr. Because of this, when a pull request is created, the owner of the repository will review, and put it in automated testing to make sure,
+* The change will build on various OS platforms (Ubuntu, RHEL, etc.)
+* The build package will install and run the code on different GPU architectures (MI-series, Radeon series cards, etc.),
+* And the test results will achieve the goal as expected.
+
+## Code Structure ##
+
+clr contains three parts of codes,
+- `hipamd` - contains implementation for HIP runtime on the AMD platform, which includes
+   - `include/hip/amd_detail` for headers
+   - `/src` for all types of functionality implementation such as hip event, memory, module and texture, etc.
+
+- `opencl` - contains implementation of OpenCL on the AMD platform.
+
+- `rocclr` - contains compute runtime used in HIP and OpenCL, which includes
+   - `include`, header files,
+   - `device`, implementation of GPU device related interfaces to the backend support,
+   - `cimpiler`, implementation of interfaces with compiler,
+   - `utils`, implementation of some useful utilities,
+   - `os`, implementation of OS related interfaces.
+
+
+## Coding Style ##
+
+clr is a C++ runtime API implementation on the AMD platform. It allows codeing in C++ programming language, and follows styles as below,
+- Code Indentation:
+    - Tabs should be expanded to spaces.
+    - Use 4 spaces indentation.
+- Capitalization and Naming
+    - Prefer camelCase for HIP interfaces and internal symbols.  Note `HIP_CLANG` uses `_` for separator.
+      This guideline is not yet consistently followed in HIP code - eventual compliance is aspirational.
+    - Member variables should begin with a leading "_".  This allows them to be easily distinguished from other variables or functions.
+
+- `{}` placement
+    - namespace should be on same line as `{` and separated by a space.
+    - Single-line if statement should still use `{/}` pair (even though C++ does not require).
+    - For functions, the opening `{` should be placed on a new line.
+    - For if/else blocks, the opening `{` is placed on same line as the if/else. Use a space to separate `{` from if/else. For example,
+```console
+    if (foo) {
+        doFoo()
+    } else {
+        doFooElse();
+    }
+```
+
+- Miscellaneous
+    - All references in function parameter lists should be const.
+    - "ihip" means internal hip structures.  These should not be exposed through the HIP API.
+    - Keyword TODO refers to a note that should be addressed in long-term.  Could be style issue, software architecture, or known bugs.
+    - FIXME refers to a short-term bug that needs to be addressed.
+
+- `HIP_INIT_API()` should be placed at the start of each top-level HIP API.  This function will make sure the HIP runtime is initialized, and also constructs an appropriate API string for tracing and CodeXL marker tracing. The arguments to HIP_INIT_API should match those of the parent function.
+- `hipExtGetLastError()` can be called as the AMD platform specific API, to return error code from last HIP API called from the active host thread. `hipGetLastError()` and `hipPeekAtLastError()` can also return the last error that was returned by any of the HIP runtime calls in the same host thread.
+- All HIP environment variables should begin with the keyword HIP_
+Environment variables should be long enough to describe their purpose but short enough so they can be remembered - perhaps 10-20 characters, with 3-4 parts separated by underscores.
+To see the list of current environment variables, along with their values, set HIP_PRINT_ENV and run any hip applications on ROCm platform.
+HIPCC or other tools may support additional environment variables which should follow the above convention.
+
+## Pull Request Guidelines ##
+
+By creating a pull request, you agree to the statements made in the code license section. Your pull request should target the default branch. Our current default branch is the develop branch, which serves as our integration branch.
+
+Follow existing best practice for writing a good Git commit message.
+
+Some tips:
+    http://chris.beams.io/posts/git-commit/
+    https://robots.thoughtbot.com/5-useful-tips-for-a-better-commit-message
+
+In particular :
+   - Use imperative voice, ie "Fix this bug", "Refactor the XYZ routine", "Update the doc".
+     Not : "Fixing the bug", "Fixed the bug", "Bug fix", etc.
+   - Subject should summarize the commit.  Do not end subject with a period.  Use a blank line
+     after the subject.
+
+### Deliverables ###
+
+HIP is an open source library. Because of this, we include the following license description at the top of every source file.
+If you create new source files in the repository, please include this text in them as well (replacing "xx" with the digits for the current year):
+```
+// Copyright (c) 20xx Advanced Micro Devices, Inc. All rights reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+```
+
+### Process ###
+
+After you create a PR, you can take a look at a diff of the changes you made using the PR's "Files" tab.
+
+PRs must pass through the checks and the code review described in the [Acceptance Criteria](#acceptance-criteria) section before they can be merged.
+
+Checks may take some time to complete. You can view their progress in the table near the bottom of the pull request page. You may also be able to use the links in the table
+to view logs associated with a check if it fails.
+
+During code reviews, another developer will take a look through your proposed change. If any modifications are requested (or further discussion about anything is
+needed), they may leave a comment. You can follow up and respond to the comment, and/or create comments of your own if you have questions or ideas.
+When a modification request has been completed, the conversation thread about it will be marked as resolved.
+
+To update the code in your PR (eg. in response to a code review discussion), you can simply push another commit to the branch used in your pull request.
+

projects/clr/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2008 - 2025 Advanced Micro Devices, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

projects/clr/README.md

@@ -0,0 +1,68 @@
+# AMD CLR - Compute Language Runtimes
+
+AMD CLR (Compute Language Runtime) contains source codes for AMD's compute languages runtimes: `HIP` and `OpenCL™`.
+
+## Project Organisation
+
+- `hipamd` - contains implementation of `HIP` language on AMD platform. It is hosted at [ROCm/clr/hipamd](https://github.com/ROCm/clr/tree/develop/hipamd)
+- `opencl` - contains implementation of [OpenCL™](https://www.khronos.org/opencl/) on AMD platform. Now it is hosted at [ROCm/clr/opencl](https://github.com/ROCm/clr/tree/develop/opencl)
+- `rocclr` - contains compute runtime used in `HIP` and `OpenCL™`. This is hosted at [ROCm/clr/rocclr](https://github.com/ROCm/clr/tree/develop/rocclr)
+
+## How to build/install
+
+### Prerequisites
+
+Please refer to Quick Start Guide in [ROCm Docs](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html).
+
+Building clr requires `rocm-hip-libraries` meta package, which provides the pre-requisites for clr.
+If you need to build static clr library, `rocm-llvm-dev` package should be installed which has support for compilation of the static library.
+
+### Linux
+
+- Clone this repository
+
+- `cd clr && mkdir build && cd build`
+
+- For HIP:
+
+    - `cmake .. -DCLR_BUILD_HIP=ON -DHIP_COMMON_DIR=$HIP_COMMON_DIR -DHIPCC_BIN_DIR=$HIPCC_BIN_DIR`
+
+        - `HIP_COMMON_DIR` points to [HIP](https://github.com/ROCm/HIP)
+
+        - `HIPCC_BIN_DIR` points to hipcc directory, if you have ROCm installed you can point it to `$ROCM_PATH/bin`
+
+- For OpenCL™:
+
+    - `cmake .. -DCLR_BUILD_OCL=ON`
+
+- Build and install
+
+```
+make
+make install
+```
+
+Users can also build `OCL` and `HIP` at the same time by passing `-DCLR_BUILD_HIP=ON -DCLR_BUILD_OCL=ON` to configure command.
+
+HIP/CLR can be built as a static library, users need to add more options in `cmake` command,
+ `-DBUILD_SHARED_LIBS=OFF` and `-DCMAKE_PREFIX_PATH="/opt/rocm/;/opt/rocm/llvm`.
+
+For detail instructions, please refer to [how to build HIP](https://rocm.docs.amd.com/projects/HIP/en/latest/install/build.html)
+
+## Tests
+
+`hip-tests` is a separate repository hosted at [hip-tests](https://github.com/ROCm/hip-tests).
+
+To run `hip-tests` please go to the repository and follow the steps.
+
+## Release notes
+
+HIP provides release notes in [CLR change log](./CHANGELOG.md), which has the records of changes in each release.
+
+## Disclaimer
+
+The information presented in this document is for informational purposes only and may contain technical inaccuracies, omissions, and typographical errors. The information contained herein is subject to change and may be rendered inaccurate for many reasons, including but not limited to product and roadmap changes, component and motherboard versionchanges, new model and/or product releases, product differences between differing manufacturers, software changes, BIOS flashes, firmware upgrades, or the like. Any computer system has risks of security vulnerabilities that cannot be completely prevented or mitigated.AMD assumes no obligation to update or otherwise correct or revise this information. However, AMD reserves the right to revise this information and to make changes from time to time to the content hereof without obligation of AMD to notify any person of such revisions or changes.THIS INFORMATION IS PROVIDED ‘AS IS.” AMD MAKES NO REPRESENTATIONS OR WARRANTIES WITH RESPECT TO THE CONTENTS HEREOF AND ASSUMES NO RESPONSIBILITY FOR ANY INACCURACIES, ERRORS, OR OMISSIONS THAT MAY APPEAR IN THIS INFORMATION. AMD SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR ANY PARTICULAR PURPOSE. IN NO EVENT WILL AMD BE LIABLE TO ANY PERSON FOR ANY RELIANCE, DIRECT, INDIRECT, SPECIAL, OR OTHER CONSEQUENTIAL DAMAGES ARISING FROM THE USE OF ANY INFORMATION CONTAINED HEREIN, EVEN IF AMD IS EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies.
+
+© 2023 Advanced Micro Devices, Inc. All Rights Reserved.
+
+OpenCL™ is registered Trademark of Apple

projects/clr/hipamd/.gitattributes

@@ -0,0 +1,20 @@
+# Set the default behavior, in case people don't have core.autolf set.
+* text=auto
+
+# Explicitly declare text files you want to always be normalized and converted
+# to have LF line endings on checkout.
+*.c text eol=lf
+*.cpp text eol=lf
+*.cc text eol=lf
+*.h text eol=lf
+*.hpp text eol=lf
+*.txt text eol=lf
+
+# Define files to support auto-remove trailing white space
+# Need to run the command below, before add modified file(s) to the staging area
+# git config filter.trimspace.clean 'sed -e "s/[[:space:]]*$//g"'
+*.cpp filter=trimspace
+*.c filter=trimspace
+*.h filter=trimspacecpp
+*.hpp filter=trimspace
+*.md filter=trimspace

projects/clr/hipamd/.gitignore

@@ -0,0 +1,16 @@
+.*
+!.gitignore
+*.o
+*.exe
+*.swp
+lib
+packages
+build
+bin/hipInfo
+bin/hipBusBandwidth
+bin/hipDispatchLatency
+bin/hipify-clang
+tags
+samples/1_Utils/hipInfo/hipInfo
+samples/1_Utils/hipBusBandwidth/hipBusBandwidth
+samples/1_Utils/hipDispatchLatency/hipDispatchLatency

projects/clr/hipamd/CMakeLists.txt

@@ -0,0 +1,484 @@
+# Copyright (c) 2016 - 2021 Advanced Micro Devices, Inc. All Rights Reserved.
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+project(hip)
+
+include(GNUInstallDirs)
+
+# sample command for hip-rocclr runtime, you'll need to have rocclr built
+# ROCM_PATH is the path where ROCM is installed
+#  For shared lib of hip-rocclr runtime
+#    For release version
+#      cmake -DHIP_COMMON_DIR="$HIP_DIR" -DHIPCC_BIN_DIR="$HIPCC_DIR/bin" -DAMD_OPENCL_PATH=$OPENCL_DIR -DROCCLR_PATH=$ROCCLR_DIR -DCMAKE_PREFIX_PATH="<ROCM_PATH>/" -DCMAKE_INSTALL_PREFIX=</where/to/install/hip> ..
+#    For debug version
+#      cmake -DHIP_COMMON_DIR="$HIP_DIR" -DHIPCC_BIN_DIR="$HIPCC_DIR/bin" -DAMD_OPENCL_PATH=$OPENCL_DIR -DROCCLR_PATH=$ROCCLR_DIR -DCMAKE_PREFIX_PATH="<ROCM_PATH>/" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=</where/to/install/hip> ..
+#  For static lib of hip-rocclr runtime
+#    For release version
+#      cmake -DHIP_COMMON_DIR="$HIP_DIR" -DHIPCC_BIN_DIR="$HIPCC_DIR/bin" -DAMD_OPENCL_PATH=$OPENCL_DIR -DROCCLR_PATH=$ROCCLR_DIR -DBUILD_SHARED_LIBS=OFF -DCMAKE_PREFIX_PATH="<ROCM_PATH>/" -DCMAKE_INSTALL_PREFIX=</where/to/install/hip> ..
+#    For debug version
+#      cmake -DHIP_COMMON_DIR="$HIP_DIR" -DHIPCC_BIN_DIR="$HIPCC_DIR/bin" -DAMD_OPENCL_PATH=$OPENCL_DIR -DROCCLR_PATH=$ROCCLR_DIR -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH="<ROCM_PATH>/" -DCMAKE_INSTALL_PREFIX=</where/to/install/hip> ..
+#  If you don't specify CMAKE_INSTALL_PREFIX, hip-rocclr runtime will be installed to "<ROCM_PATH>/hip".
+#  By default, CMake will search for a folder named vdi or ROCclr relative to the current path. Specify -DROCCLR_PATH=$ROCCLR_DIR if rocclr source is in obscure location.
+#  By default, CMake will search for a folder named opencl or ROCm-OpenCL-Runtime relative to the current path. Specify -DAMD_OPENCL_PATH=$OPENCL_DIR if opencl source is in obscure location.
+list(APPEND CMAKE_MODULE_PATH ${HIP_COMMON_DIR}/cmake)
+
+
+#############################
+# Options
+#############################
+option(__HIP_ENABLE_PCH "Enable/Disable pre-compiled hip headers" ON)
+option(HIP_OFFICIAL_BUILD "Enable/Disable for mainline/staging builds" OFF)
+option(BUILD_SHARED_LIBS "Build the shared library" ON)
+
+if(MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi")
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG:FULL")
+endif()
+
+# Set default HIPCC_BIN_DIR to /opt/rocm/bin
+set(HIPCC_BIN_DIR "/opt/rocm/bin" CACHE STRING "HIPCC and HIPCONFIG binary directories")
+
+if(__HIP_ENABLE_PCH)
+  set(_pchStatus 1)
+else()
+  set(_pchStatus 0)
+endif()
+
+message(STATUS "HIPCC_BIN_DIR found at ${HIPCC_BIN_DIR}")
+message(STATUS "HIP_COMMON_DIR found at ${HIP_COMMON_DIR}")
+message(STATUS "HIPNV_DIR found at ${HIPNV_DIR}")
+set(HIP_COMMON_INCLUDE_DIR ${HIP_COMMON_DIR}/include)
+set(HIP_COMMON_BIN_DIR ${HIP_COMMON_DIR}/bin)
+if (WIN32)
+  set(HIPCC_EXECUTABLE "hipcc.exe")
+  set(HIPCONFIG_EXECUTABLE "hipconfig.exe")
+else()
+  set(HIPCC_EXECUTABLE "hipcc")
+  set(HIPCONFIG_EXECUTABLE "hipconfig")
+endif()
+
+#############################
+# Setup config generation
+#############################
+string(TIMESTAMP _timestamp UTC)
+set(_versionInfo "# Auto-generated by cmake\n")
+set(_buildInfo "# Auto-generated by cmake on ${_timestamp} UTC\n")
+macro(add_to_config _configfile _variable)
+    set(${_configfile} "${${_configfile}}${_variable}=${${_variable}}\n")
+endmacro()
+
+#############################
+# Setup version information
+#############################
+find_package(Perl REQUIRED)
+
+# Determine HIP_BASE_VERSION
+set(ENV{HIP_PATH} "")
+file(STRINGS ${HIP_COMMON_DIR}/VERSION VERSION_LIST REGEX "^[0-9]+")
+list(GET VERSION_LIST 0 HIP_VERSION_MAJOR)
+list(GET VERSION_LIST 1 HIP_VERSION_MINOR)
+list(GET VERSION_LIST 2 HIP_VERSION_PATCH)
+set(HIP_VERSION_GITDATE 0)
+
+find_package(Git)
+
+# FIXME: Two different version strings used.
+# Below we use UNIX commands, not compatible with Windows.
+if(GIT_FOUND)
+  # use the commit date, instead of build date
+  execute_process(COMMAND ${GIT_EXECUTABLE} show -s --format=%ct
+    RESULT_VARIABLE git_result
+    OUTPUT_VARIABLE git_output
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(git_result EQUAL 0)
+    set(HIP_VERSION_UNIXDATE ${git_output})
+  endif()
+
+  # get date information based on UTC
+  # use the last two digits of year + week number + day in the week as HIP_VERSION_GITDATE
+  execute_process(COMMAND ${PERL_EXECUTABLE} "-MPOSIX=strftime" "-le" "print strftime \'%y%W%w\',gmtime(${HIP_VERSION_UNIXDATE})"
+    RESULT_VARIABLE git_result
+    OUTPUT_VARIABLE git_output
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(git_result EQUAL 0)
+    set(HIP_VERSION_GITDATE ${git_output})
+  endif()
+
+  # get commit short hash
+  execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+    RESULT_VARIABLE git_result
+    OUTPUT_VARIABLE git_output
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(git_result EQUAL 0)
+    set(HIP_VERSION_GITHASH ${git_output})
+  endif()
+
+  set(HIP_VERSION_BUILD_ID 0)
+  set(HIP_VERSION_BUILD_NAME "")
+  if(NOT DEFINED ENV{HIP_OFFICIAL_BUILD} AND NOT HIP_OFFICIAL_BUILD)
+    set(HIP_VERSION_PATCH ${HIP_VERSION_GITDATE})
+  endif()
+
+  if(DEFINED ENV{ROCM_LIBPATCH_VERSION})
+    set(HIP_PACKAGING_VERSION_PATCH ${HIP_VERSION_PATCH}.$ENV{ROCM_LIBPATCH_VERSION})
+  else()
+    set(HIP_PACKAGING_VERSION_PATCH ${HIP_VERSION_PATCH}-${HIP_VERSION_GITHASH})
+  endif()
+else()
+  set(HIP_VERSION_BUILD_ID 0)
+  set(HIP_VERSION_BUILD_NAME "")
+  # FIXME: Some parts depend on this being set.
+  set(HIP_PACKAGING_VERSION_PATCH "0")
+endif()
+
+## Debian package specific variables
+if ( DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE} )
+  set ( CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE} )
+else()
+  set ( CPACK_DEBIAN_PACKAGE_RELEASE "local" )
+endif()
+message (STATUS "Using CPACK_DEBIAN_PACKAGE_RELEASE ${CPACK_DEBIAN_PACKAGE_RELEASE}" )
+
+## RPM package specific variables
+if ( DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE} )
+  set ( CPACK_RPM_PACKAGE_RELEASE $ENV{CPACK_RPM_PACKAGE_RELEASE} )
+else()
+  set ( CPACK_RPM_PACKAGE_RELEASE "local" )
+endif()
+
+set ( EL7_DISTRO "FALSE" )
+## 'dist' breaks manual builds on debian systems due to empty Provides
+execute_process( COMMAND rpm --eval %{?dist}
+                 RESULT_VARIABLE PROC_RESULT
+                 OUTPUT_VARIABLE EVAL_RESULT
+                 OUTPUT_STRIP_TRAILING_WHITESPACE )
+
+if ( PROC_RESULT EQUAL "0" AND NOT EVAL_RESULT STREQUAL "" )
+  string ( APPEND CPACK_RPM_PACKAGE_RELEASE "%{?dist}" )
+  if ( "${EVAL_RESULT}" STREQUAL ".el7" )
+     set ( EL7_DISTRO "TRUE" )
+  endif() # end check string el7
+endif()
+message(STATUS "CPACK_RPM_PACKAGE_RELEASE: ${CPACK_RPM_PACKAGE_RELEASE}")
+
+set (DEB10_DISTRO "FALSE")
+execute_process( COMMAND sh "-c" "cat /etc/os-release | grep \"^NAME=\""
+                 RESULT_VARIABLE PROC_RESULT_NAME
+                 OUTPUT_VARIABLE OS_NAME_RESULT
+                 OUTPUT_STRIP_TRAILING_WHITESPACE )
+execute_process( COMMAND sh "-c" "cat /etc/os-release | grep \"^VERSION_ID=\""
+                 RESULT_VARIABLE PROC_RESULT_VERSION
+                 OUTPUT_VARIABLE OS_VERSION_ID_RESULT
+                 OUTPUT_STRIP_TRAILING_WHITESPACE )
+
+if ( PROC_RESULT_NAME EQUAL "0" AND NOT OS_NAME_RESULT STREQUAL "" AND
+     PROC_RESULT_VERSION EQUAL "0" AND NOT OS_VERSION_ID_RESULT STREQUAL "")
+  if ( "${OS_NAME_RESULT}" STREQUAL "NAME=\"Debian GNU/Linux\"" AND "${OS_VERSION_ID_RESULT}" STREQUAL "VERSION_ID=\"10\"")
+     set ( DEB10_DISTRO "TRUE" )
+  endif()
+endif()
+
+add_to_config(_versionInfo HIP_PACKAGING_VERSION_PATCH)
+add_to_config(_versionInfo CPACK_DEBIAN_PACKAGE_RELEASE)
+add_to_config(_versionInfo CPACK_RPM_PACKAGE_RELEASE)
+
+add_to_config(_versionInfo HIP_VERSION_MAJOR)
+add_to_config(_versionInfo HIP_VERSION_MINOR)
+add_to_config(_versionInfo HIP_VERSION_PATCH)
+add_to_config(_versionInfo HIP_VERSION_GITHASH)
+
+set (HIP_LIB_VERSION_MAJOR ${HIP_VERSION_MAJOR})
+set (HIP_LIB_VERSION_MINOR ${HIP_VERSION_MINOR})
+if (${ROCM_PATCH_VERSION} )
+   set (HIP_LIB_VERSION_PATCH ${ROCM_PATCH_VERSION})
+elseif (DEFINED HIP_VERSION_GITHASH)
+   set (HIP_LIB_VERSION_PATCH ${HIP_VERSION_PATCH}-${HIP_VERSION_GITHASH})
+else ()
+   set (HIP_LIB_VERSION_PATCH ${HIP_VERSION_PATCH})
+endif ()
+set (HIP_LIB_VERSION_STRING "${HIP_LIB_VERSION_MAJOR}.${HIP_LIB_VERSION_MINOR}.${HIP_LIB_VERSION_PATCH}")
+
+# overwrite HIP_VERSION_PATCH for packaging
+set(HIP_VERSION ${HIP_VERSION_MAJOR}.${HIP_VERSION_MINOR}.${HIP_PACKAGING_VERSION_PATCH})
+
+# Remove when CI is updated
+if(HIP_PLATFORM STREQUAL "rocclr")
+   set(HIP_PLATFORM "amd")
+endif()
+#############################
+# Configure variables
+#############################
+# Determine HIP_PLATFORM
+if(NOT DEFINED HIP_PLATFORM)
+    if(NOT DEFINED ENV{HIP_PLATFORM})
+        execute_process(COMMAND ${__HIPCONFIG_EXECUTABLE__} --platform
+            OUTPUT_VARIABLE HIP_PLATFORM
+            OUTPUT_STRIP_TRAILING_WHITESPACE)
+    else()
+        set(HIP_PLATFORM $ENV{HIP_PLATFORM} CACHE STRING "HIP Platform")
+    endif()
+endif()
+message(STATUS "HIP Platform: " ${HIP_PLATFORM})
+
+if(HIP_PLATFORM STREQUAL "nvidia")
+    set(HIP_RUNTIME "cuda"  CACHE STRING "HIP Runtime")
+    set(HIP_COMPILER "nvcc" CACHE STRING "HIP Compiler")
+    if (NOT EXISTS ${HIPNV_DIR})
+      message(FATAL_ERROR "Invalid path HIPNV_DIR=${HIPNV_DIR}, please pass valid path -DHIPNV_DIR=<path>" )
+    endif()
+elseif(HIP_PLATFORM STREQUAL "amd")
+    set(HIP_RUNTIME "rocclr" CACHE STRING "HIP Runtime")
+    set(HIP_COMPILER "clang" CACHE STRING "HIP Compiler")
+else()
+    message(FATAL_ERROR "Unexpected HIP_PLATFORM: " ${HIP_PLATFORM})
+endif()
+
+message(STATUS "HIP Runtime: " ${HIP_RUNTIME})
+message(STATUS "HIP Compiler: " ${HIP_COMPILER})
+
+add_to_config(_buildInfo HIP_RUNTIME)
+add_to_config(_buildInfo HIP_COMPILER)
+
+if (NOT DEFINED ROCM_PATH )
+     set ( ROCM_PATH "/opt/rocm"  CACHE STRING "Default ROCM installation directory." )
+endif ()
+message (STATUS "ROCM Installation path(ROCM_PATH): ${ROCM_PATH}")
+
+# Determine HIP install path
+if (UNIX)
+    set(HIP_DEFAULT_INSTALL_PREFIX "${ROCM_PATH}")
+endif()
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+    set(CMAKE_INSTALL_PREFIX ${HIP_DEFAULT_INSTALL_PREFIX} CACHE PATH "Installation path for HIP" FORCE)
+endif()
+
+if(DEV_LOG_ENABLE MATCHES "yes")
+  add_definitions(-DDEV_LOG_ENABLE)
+endif()
+
+# Set default install path as "${ROCM_PATH}", can override the path from cmake build.
+set(CPACK_INSTALL_PREFIX ${HIP_DEFAULT_INSTALL_PREFIX} CACHE PATH "Package Installation path for HIP")
+
+if(IS_ABSOLUTE ${CMAKE_INSTALL_PREFIX})
+    message(STATUS "HIP will be installed in: " ${CMAKE_INSTALL_PREFIX})
+else()
+    message(FATAL_ERROR "Don't know where to install HIP. Please specify absolute path using -DCMAKE_INSTALL_PREFIX")
+endif()
+
+# set the installation path for the installer package
+set(CPACK_SET_DESTDIR ON CACHE BOOL "Installer package will install hip to CMAKE_INSTALL_PREFIX instead of CPACK_PACKAGING_INSTALL_PREFIX")
+if (NOT CPACK_SET_DESTDIR)
+  set(CPACK_PACKAGING_INSTALL_PREFIX "${ROCM_PATH}" CACHE PATH "Default installation path of hcc installer package")
+endif (NOT CPACK_SET_DESTDIR)
+
+#############################
+# Build steps
+#############################
+set(BIN_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR})
+set(LIB_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
+set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR})
+set(CONFIG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip)
+set(CONFIG_LANG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip-lang)
+set(CONFIG_RTC_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hiprtc)
+
+# Generate hip_version.h
+set(_versionInfoHeader
+"// Auto-generated by cmake\n
+#ifndef HIP_VERSION_H
+#define HIP_VERSION_H\n
+#define HIP_VERSION_MAJOR ${HIP_VERSION_MAJOR}
+#define HIP_VERSION_MINOR ${HIP_VERSION_MINOR}
+#define HIP_VERSION_PATCH ${HIP_VERSION_PATCH}
+#define HIP_VERSION_GITHASH \"${HIP_VERSION_GITHASH}\"
+#define HIP_VERSION_BUILD_ID ${HIP_VERSION_BUILD_ID}
+#define HIP_VERSION_BUILD_NAME \"${HIP_VERSION_BUILD_NAME}\"
+#define HIP_VERSION    (HIP_VERSION_MAJOR * 10000000 + HIP_VERSION_MINOR * 100000 + HIP_VERSION_PATCH)\n
+#define __HIP_HAS_GET_PCH ${_pchStatus}\n
+#endif\n
+")
+file(WRITE "${PROJECT_BINARY_DIR}/include/hip/hip_version.h" ${_versionInfoHeader})
+
+# Generate .hipInfo
+file(WRITE "${PROJECT_BINARY_DIR}/share/hip/.hipInfo" ${_buildInfo})
+
+# Generate version
+file(WRITE "${PROJECT_BINARY_DIR}/share/hip/version" ${_versionInfo})
+
+if(HIP_RUNTIME STREQUAL "rocclr")
+   add_subdirectory(src)
+endif()
+
+# Build doxygen documentation
+find_program(DOXYGEN_EXE doxygen)
+if(DOXYGEN_EXE)
+    if(EXISTS "${HIP_COMMON_DIR}/docs/doxygen-input/doxy.cfg")
+        add_custom_target(doc COMMAND HIP_PATH=${CMAKE_CURRENT_SOURCE_DIR} ${DOXYGEN_EXE} ${HIP_COMMON_DIR}/docs/doxygen-input/doxy.cfg
+            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/docs)
+    elseif(EXISTS "${HIP_COMMON_DIR}/docs/.doxygen/Doxyfile")
+        add_custom_target(doc COMMAND HIP_PATH=${CMAKE_CURRENT_SOURCE_DIR} ${DOXYGEN_EXE} ${HIP_COMMON_DIR}/docs/.doxygen/Doxyfile
+            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/docs)
+    else()
+        message(STATUS "Unable to find doxygen config file. Will not generate doxygen output")
+    endif()
+endif()
+
+#############################
+# Install steps
+#############################
+
+# Install .hipInfo
+install(FILES ${PROJECT_BINARY_DIR}/share/hip/.hipInfo DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+# Install version
+install(FILES ${PROJECT_BINARY_DIR}/share/hip/version DESTINATION ${CMAKE_INSTALL_DATADIR}/hip)
+# .hipVersion is added to satisfy Windows compute build.
+#TODO to be removed
+if(WIN32)
+    install(FILES ${PROJECT_BINARY_DIR}/share/hip/version DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME .hipVersion)
+endif()
+
+# Install src, bin, include & cmake if necessary
+execute_process(COMMAND test ${CMAKE_INSTALL_PREFIX} -ef ${CMAKE_CURRENT_SOURCE_DIR}
+    RESULT_VARIABLE INSTALL_SOURCE)
+if(NOT ${INSTALL_SOURCE} EQUAL 0)
+    if(WIN32)
+      install(DIRECTORY ${HIP_COMMON_BIN_DIR} DESTINATION . USE_SOURCE_PERMISSIONS)
+      if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+        install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/src/" DESTINATION ${CMAKE_INSTALL_BINDIR}
+                FILES_MATCHING PATTERN "*.pdb"
+                               PATTERN "*.ilk"
+                               PATTERN "CMakeFiles" EXCLUDE
+                               PATTERN "hip_rtc_gen" EXCLUDE
+                               PATTERN "libelf" EXCLUDE
+                               PATTERN "loader" EXCLUDE
+                               PATTERN "pal" EXCLUDE
+                               PATTERN "libamdhsacode" EXCLUDE)
+      endif()
+    else()
+      # Exclude .bat files on Linux.
+      #Hip bin files moved to /opt/rocm/bin and the file permission need to set properly
+      install(DIRECTORY ${HIP_COMMON_BIN_DIR} DESTINATION . USE_SOURCE_PERMISSIONS
+              DIRECTORY_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+              PATTERN *.bat EXCLUDE)
+    endif()
+
+    if(WIN32) #not required for flat folder structure
+      # The following two lines will be removed after upstream updation
+      install(CODE "MESSAGE(\"Removing ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}\")")
+      install(CODE "file(REMOVE_RECURSE ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR})")
+    endif()
+
+    install(DIRECTORY include DESTINATION .)
+    if(DEFINED HIPNV_DIR)
+        install(DIRECTORY ${HIPNV_DIR}/include/hip/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip/)
+    endif()
+    install(DIRECTORY ${HIP_COMMON_INCLUDE_DIR}/hip/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip/)
+    if(WIN32)
+      install(DIRECTORY ${HIP_COMMON_DIR}/cmake DESTINATION .)
+    else()
+      install(DIRECTORY ${HIP_COMMON_DIR}/cmake/ DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR})
+    endif()
+endif()
+
+# Install generated headers
+# FIXME: Associate with individual targets.
+if(HIP_PLATFORM STREQUAL "amd")
+install(FILES ${PROJECT_BINARY_DIR}/include/hip/amd_detail/hip_prof_str.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip/amd_detail)
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin DESTINATION . USE_SOURCE_PERMISSIONS)
+endif()
+install(FILES ${PROJECT_BINARY_DIR}/include/hip/hip_version.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip)
+
+set(HIP_INSTALLS_HIPCC OFF)
+if (NOT ${HIPCC_BIN_DIR} STREQUAL "")
+  file(TO_CMAKE_PATH "${HIPCC_BIN_DIR}" HIPCC_BIN_DIR)
+  if(EXISTS ${HIPCC_BIN_DIR})
+    set(HIP_INSTALLS_HIPCC ON)
+    install(PROGRAMS ${HIPCC_BIN_DIR}/${HIPCC_EXECUTABLE} DESTINATION bin)
+    install(PROGRAMS ${HIPCC_BIN_DIR}/${HIPCONFIG_EXECUTABLE} DESTINATION bin)
+
+    if(NOT UNIX)
+      install(PROGRAMS ${HIPCC_BIN_DIR}/hipcc.bat DESTINATION bin)
+      install(PROGRAMS ${HIPCC_BIN_DIR}/hipconfig.bat DESTINATION bin)
+    endif()
+  endif()
+endif()
+
+#############################
+# hip-config
+#############################
+include(CMakePackageConfigHelpers)
+
+configure_package_config_file(
+    hip-config.cmake.in
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-config.cmake
+    INSTALL_DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR}
+    PATH_VARS LIB_INSTALL_DIR INCLUDE_INSTALL_DIR BIN_INSTALL_DIR
+    )
+
+configure_package_config_file(
+      hip-config-amd.cmake.in
+      ${CMAKE_CURRENT_BINARY_DIR}/hip-config-amd.cmake
+      INSTALL_DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR}
+      PATH_VARS LIB_INSTALL_DIR INCLUDE_INSTALL_DIR BIN_INSTALL_DIR
+      )
+
+configure_package_config_file(
+      hip-config-nvidia.cmake
+      ${CMAKE_CURRENT_BINARY_DIR}/hip-config-nvidia.cmake
+      INSTALL_DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR}
+      PATH_VARS LIB_INSTALL_DIR INCLUDE_INSTALL_DIR BIN_INSTALL_DIR
+      )
+
+write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-config-version.cmake
+    VERSION "${HIP_VERSION_MAJOR}.${HIP_VERSION_MINOR}.${HIP_VERSION_PATCH}"
+    COMPATIBILITY SameMajorVersion
+    )
+install(
+    FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-config.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-config-amd.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-config-nvidia.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-config-version.cmake
+    DESTINATION
+    ${CONFIG_PACKAGE_INSTALL_DIR}
+    )
+# Packaging invokes UNIX commands, which are not available on Windows.
+
+if(NOT WIN32)
+    add_subdirectory(packaging)
+endif()
+
+#############################
+# Code analysis
+#############################
+# Target: clang
+if(HIP_HIPCC_EXECUTABLE)
+    add_custom_target(analyze
+        COMMAND ${HIP_HIPCC_EXECUTABLE} -fvisibility=hidden -fvisibility-inlines-hidden --analyze --analyzer-outputtext  -isystem ${ROCM_PATH}/${CMAKE_INSTALL_INCLUDEDIR} -Wno-unused-command-line-argument -I${ROCM_PATH}/${CMAKE_INSTALL_INCLUDEDIR} -c  src/*.cpp -Iinclude/ -I./
+    WORKING_DIRECTORY ${HIP_SRC_PATH})
+    if(CPPCHECK_EXE)
+        add_dependencies(analyze cppcheck)
+    endif()
+endif()

projects/clr/hipamd/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008 - 2022 Advanced Micro Devices, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

projects/clr/hipamd/bin/roc-obj

@@ -0,0 +1,267 @@
+#!/bin/bash
+
+#| Usage: roc-obj [-h] [-t REGEXP] [-o OUTDIR] [-I REPLACE-STRING|-i] [-d]
+#|                EXECUTABLE... [: [SUFFIX COMMAND [ARGS...] ;]...]
+#|
+#| Wrapper for roc-obj-ls and roc-obj-extract which extracts code objects
+#| embedded in each EXECUTABLE and optionally applies COMMANDs to them.
+#|
+#| If the POSIX extended regular expression REGEXP is specified, only embedded
+#| code objects whose Target ID matches REGEXP are extracted; otherwise all
+#| code objects are extracted.
+#|
+#| If the directory path OUTDIR is specified, it is created if it does not
+#| already exist, and the code objects are extracted into it; otherwise they
+#| are extracted into the current working directory.
+#|
+#| The extracted files are named by appending a ":" followed by the Target ID
+#| of the extracted code object to the input filename EXECUTABLE they were
+#| extracted from.
+#|
+#| If the list of EXECUTABLE arguments is terminated with ":" then after all
+#| selected files are successfully extracted, zero or more additional embedded
+#| command-lines, separated by ";", are read from the command-line starting
+#| after the ":". These must specify a SUFFIX used to name the output of the
+#| corresponding COMMAND, along with the COMMAND name and any ARGS to it.
+#|
+#| Then each COMMAND is executed, as if by a POSIX "execvp" function, once for
+#| each embedded code object that was created in OUTDIR. (Note: Typically this
+#| means the user must ensure the commands are present in at least one
+#| directory of the "PATH" environment variable.) For each execution of
+#| COMMAND:
+#|
+#| If REPLACE-STRING is specified, all instances of REPLACE-STRING in ARGS are
+#| replaced with the file path of the extracted code object before executing
+#| COMMAND.
+#|
+#| The standard input is redirected from the extracted code object.
+#|
+#| If SUFFIX is "-" the standard output is not redirected. If SUFFIX is "!" the
+#| standard output is redirected to /dev/null. Otherwise, the standard output
+#| is redirected to files named by the file path of the extracted code object
+#| with SUFFIX appended.
+#|
+#| Note: The executables roc-obj-ls, roc-obj-extract, and llvm-objdump (in the
+#| case of disassembly requested using the -d flag) are searched for in a
+#| unique way. A series of directories are searched, some conditionally, until
+#| a suitable executable is found. If all directories are searched without
+#| finding the executable, an error occurs. The first directory searched is the
+#| one containing the hard-link to the roc-obj being executed, known as the
+#| "base directory". Next, if the environment variable HIP_CLANG_PATH is set,
+#| it is searched; otherwise, the base directory path is appended with
+#| "../llvm/bin" and it is searched. Finally, the PATH is searched as if by
+#| a POSIX "execvp" function.
+#|
+#| Option Descriptions:
+#|   -h, --help              print this help text and exit
+#|   -t, --target-id         only extract code objects from EXECUTABLE whose Target ID
+#|                           matches the POSIX extended regular expression REGEXP
+#|   -o, --outdir            set the output directory, which is created if it
+#|                           does not exist
+#|   -I, --replace-string    replace all occurrences of the literal string
+#|                           REPLACE-STRING in ARGS with the input filename
+#|   -i, --replace           equivalent to -I{}
+#|   -d, --disassemble       diassemble extracted code objects; equivalent to
+#|                           : .s llvm-objdump -d - ;
+#|
+#| Example Usage:
+#|
+#| Extract all code objects embedded in a.so:
+#| $ roc-obj a.so
+#|
+#| Extract all code objects embedded in a.so, b.so, and c.so:
+#| $ roc-obj a.so b.so c.so
+#|
+#| Extract all code objects embedded in a.so with "gfx9" in their Target ID:
+#| $ roc-obj -t gfx9 a.so
+#|
+#| Extract all code objects embedded in a.so into output/ (creating it if needed):
+#| $ roc-obj -o output/ a.so
+#|
+#| Extract all code objects embedded in a.so with "gfx9" in their Target ID
+#| into output/ (creating it if needed):
+#| $ roc-obj -t gfx9 -o output/ a.so
+#|
+#| Extract all code objects embedded in a.so, and then disassemble each of them
+#| to files ending with .s:
+#| $ roc-obj -d a.so
+#|
+#| Extract all code objects embedded in a.so, and count the number of bytes in
+#| each, writing the results to files ending with .count:
+#| $ roc-obj a.so : .count wc -c
+#|
+#| Extract all code objects embedded in a.so, and inspect their ELF headers
+#| using llvm-readelf (which will not read from standard input), writing to
+#| files ending with .hdr:
+#| $ roc-obj -I'{}' a.so : .hdr llvm-readelf -h '{}'
+#|
+#| Extract all code objects embedded in a.so, and then extract each of their
+#| .text sections using llvm-objcopy (which won't read from standard input
+#| or write to standard output):
+#| $ roc-obj -I'{}' a.so : ! llvm-objcopy -O binary :only-section=.text '{}' '{}.text'
+#|
+#| Extract all code objects embedded in a.so, b.so, and c.so with target
+#| feature xnack disabled into directory out/. Then, for each:
+#| Write the size in bytes into a file ending with .count, and
+#| Write a textual description of the ELF headers to a file ending with .hdr, and
+#| Extract the .text section to a file ending with .text
+#| $ roc-obj -I'{}' -t xnack- -o out/ a.so b.so c.so : \
+#|     .count wc -c \;
+#|     .hdr llvm-readelf -h '{}' \;
+#|     ! llvm-objcopy -O binary --only-section=.text '{}' '{}.text'
+
+set -euo pipefail
+
+usage() {
+  sed -n 's/^#| \?\(.*\)$/\1/p' "$0"
+}
+
+usage_then_exit() {
+  local -r status="$1"; shift
+  usage >&$(( status ? 2 : 1 ))
+  exit "$status"
+}
+
+fail() {
+  printf "error: %s\n" "$*" >&2
+  exit 1
+}
+
+# Account for the fact that we do not necessarily put ROCm tools in the PATH,
+# nor do we have a single, unified ROCm "bin/" directory.
+#
+# Note that this is only used for roc-obj-ls, roc-obj-extract, and "shortcut"
+# options like -d, and the user can still use any copy of llvm-* by explicitly
+# invoking it with a full path, e.g. : /path/to/llvm-* ... ;
+find_rocm_executable_or_fail() {
+  local -r command="$1"; shift
+  local file
+  local searched=()
+  for dir in "$BASE_DIR" "${HIP_CLANG_PATH:-"$BASE_DIR/../llvm/bin"}"; do
+    file="$dir/$command"
+    if [[ -x $file ]]; then
+      printf "%s" "$file"
+      return
+    else
+      searched+=("$dir")
+    fi
+  done
+  if hash "$command" 2>/dev/null; then
+    printf "%s" "$command"
+  else
+    fail could not find "$command" in "${searched[*]}" or PATH
+  fi
+}
+
+# Extract the embedded code objects of the executable file given as the first
+# argument into OPT_OUTDIR, filtering them via OPT_TARGET_ID.
+#
+# Deletes any resulting files which are empty, and prints the paths of the
+# remaining files.
+extract() {
+  local -r executable="$1"; shift
+  local prefix
+  prefix="$(basename -- "$executable")"
+  # We want the shell to split the result of roc-obj-ls on whitespace, as
+  # neither the Target ID nor the URI can have embedded spaces.
+  # shellcheck disable=SC2046
+  set -- $("$ROC_OBJ_LS" -- "$executable" | awk "\$2~/$OPT_TARGET_ID/")
+  while (( $# )); do
+    local output="$prefix:$1"; shift
+    output="$output.$1"; shift
+    local uri="$1"; shift
+    [[ -n $OPT_OUTDIR ]] && output="$OPT_OUTDIR/$output"
+    "$ROC_OBJ_EXTRACT" -o - -- "$uri" >"$output"
+    if [[ -s $output ]]; then
+      printf '%s\n' "$output"
+    else
+      rm "$output"
+    fi
+  done
+  (( $# )) && fail expected even number of fields from roc-obj-ls
+}
+
+# Run a command over a list of inputs, naming output files with the supplied
+# suffix and applying OPT_REPLACE_STRING if needed.
+#
+# Arguments are of the form:
+# $suffix $command $args... ; $inputs
+run_command() {
+  local -r suffix="$1"; shift
+  local -r command="$1"; shift
+  local args=()
+  while (( $# )); do
+    local arg="$1"; shift
+    [[ $arg == ';' ]] && break
+    args+=("$arg")
+  done
+  local inputs=("$@")
+  for input in "${inputs[@]}"; do
+    case "$suffix" in
+      '-') output=/dev/stdout;;
+      '!') output=/dev/null;;
+      *) output="$input$suffix";;
+    esac
+    "$command" "${args[@]//$OPT_REPLACE_STRING/$input}" <"$input" >"$output"
+  done
+}
+
+main() {
+  printf "Warning: The roc-obj tools have been DEPRECATED. Similar functionality is provided by llvm-objdump in the rocm-llvm package.\n"
+
+  [[ -n $OPT_OUTDIR ]] && mkdir -p "$OPT_OUTDIR"
+  local inputs=()
+  while (( $# )); do
+    local executable="$1"; shift
+    [[ $executable == : ]] && break
+    # Append the file paths extracted from $executable to $inputs
+    readarray -t -O "${#inputs[@]}" inputs < <(extract "$executable")
+  done
+  (( ${#inputs[@]} )) || fail no executables specified
+  while (( $# )); do
+    local suffix="$1"; shift
+    local command="$1"; shift
+    local args=()
+    while (( $# )); do
+      local arg="$1"; shift
+      [[ $arg == \; ]] && break
+      args+=("$arg")
+    done
+    run_command "$suffix" "$command" "${args[@]}" \; "${inputs[@]}"
+  done
+  (( OPT_DISASSEMBLE )) && run_command .s "$OBJDUMP" -d - \; "${inputs[@]}"
+}
+
+OPT_TARGET_ID=''
+OPT_OUTDIR=''
+OPT_REPLACE_STRING=''
+OPT_DISASSEMBLE=0
+! getopt -T || fail util-linux enhanced getopt required
+getopt="$(getopt -o +ht:o:I:id \
+          --long help,target-id:,outdir:,replace:,replace-default,disassemble \
+          -n roc-obj -- "$@")"
+eval set -- "$getopt"
+unset getopt
+while true; do
+  case "$1" in
+    -h | --help) usage_then_exit 0;;
+    -t | --target-id) OPT_TARGET_ID="${2//\//\\\/}"; shift 2;;
+    -o | --outdir) OPT_OUTDIR="$2"; shift 2;;
+    -I | --replace-string) OPT_REPLACE_STRING="$2"; shift 2;;
+    -i | --replace) OPT_REPLACE_STRING='{}'; shift;;
+    -d | --disassemble) OPT_DISASSEMBLE=1; shift;;
+    --) shift; break;;
+    *) usage_then_exit 1;;
+  esac
+done
+readonly -- OPT_TARGET_ID OPT_OUTDIR OPT_REPLACE_STRING OPT_DISASSEMBLE
+
+# We expect to be installed as ROCM_PATH/hip/bin/roc-obj, which means BASE_DIR
+# is ROCM_PATH/hip/bin.
+BASE_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
+(( OPT_DISASSEMBLE )) && OBJDUMP="$(find_rocm_executable_or_fail llvm-objdump)"
+ROC_OBJ_LS="$(find_rocm_executable_or_fail roc-obj-ls)"
+ROC_OBJ_EXTRACT="$(find_rocm_executable_or_fail roc-obj-extract)"
+readonly -- BASE_DIR OBJDUMP ROC_OBJ_LS ROC_OBJ_EXTRACT
+
+main "$@"

projects/clr/hipamd/bin/roc-obj-extract

@@ -0,0 +1,246 @@
+#!/usr/bin/perl
+# Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+use strict;
+use File::Copy;
+use File::Spec;
+use File::Basename;
+use File::Which;
+use Cwd 'realpath';
+use Getopt::Std;
+use List::Util qw(max);
+use URI::Escape;
+
+my $extract_range_specifier;
+my $extract_pid;
+my $extract_file;
+my $output_file;
+my $output_path;
+my $extract_offset;
+my $extract_size;
+my $pid_running;
+my $verbose=0;
+my $error=0;
+my $output_to_stdout=0;
+
+sub usage {
+  print("Usage: $0 [-o|v|h] URI... \n");
+  print("  URIs can be read from STDIN, one per line.\n");
+  print("  From the URIs specified, extracts code objects into files named: ");
+  print("<executable_name>-[pid<number>]-offset<number>-size<number>.co\n\n");
+  print("Options:\n");
+  print("  -o <path> \tPath for output. If \"-\" specified, code object is printed to STDOUT.\n");
+  print("  -v        \tVerbose output to STDOUT.\n");
+  print("  -h        \tShow this help message.\n");
+  print("\nURI syntax:\n");
+  print("\tcode_object_uri ::== file_uri | memory_uri\n");
+  print("\tfile_uri        ::== \"file://\" extract_file [ range_specifier ]\n");
+  print("\tmemory_uri      ::== \"memory://\" process_id range_specifier\n");
+  print("\trange_specifier ::== range_delimiter range_attribute [\"&\" range_attribute]\n");
+  print("\trange_delimiter ::== \"#\" | \"?\"\n");
+  print("\trange_attribute ::== [\"offset=\" number | \"size=\" number ]\n");
+  print("\textract_file    ::== URI_ENCODED_OS_FILE_PATH\n");
+  print("\tprocess_id      ::== DECIMAL_NUMBER\n");
+  print("\tnumber          ::== HEX_NUMBER \| DECIMAL_NUMBER \| OCTAL_NUMBER\n\n");
+  print("\tExample: file://dir1/dir2/hello_world#offset=133&size=14472 \n");
+  print("\t         memory://1234#offset=0x20000&size=3000\n\n");
+  print("  NOTES:\n\n");
+  print("\tWhen specifying a URI in a shell command you will need to escape the \'&\' character in the range_specifier.\n");
+  print("\tIf \"size=\" is not specified, the default is the remainder of the file from the given offset.\n\n");
+
+  exit($error);
+}
+
+# Process options
+my %options=();
+getopts('vho:', \%options);
+
+# this tool has been deprecated
+print(STDERR "Warning: This tool has been DEPRECATED. Similar functionality is provided by llvm-objdump in the rocm-llvm package.\n");
+
+if (defined $options{h}) {
+  usage();
+}
+
+if (defined $options{v}) {
+  $verbose = 1;
+}
+
+if (defined $options{o}) {
+  $output_path = $options{o};
+  if ($output_path eq "-") {
+    $output_to_stdout=1;
+  } else {
+    (-d $output_path) || die("Error: Path \'$output_path\' cannot be found.\n");
+  }
+}
+
+# Only push STDIN if there are no arguments -- otherwise this
+# consumes the caller's stdin by accident.
+# push STDIN to ARGV array.
+if ($#ARGV < 0) {
+  push @ARGV, <STDIN> unless -t STDIN;
+}
+
+# error check: enough arguments presented.
+if ($#ARGV < 0) {
+  print(STDERR "Error: No arguments.\n"); $error++;
+  usage();
+}
+
+# error check: command dd is available.
+my $dd_cmd = which("dd");
+(-f $dd_cmd) || die("Error: Can't find dd command\n");
+
+foreach my $uri_str(@ARGV) {
+  chomp $uri_str;
+
+  my ($uri_protocol, $specs) = split(/:\/\//,$uri_str);
+  my $decoded_extract_file;
+  my $file_size;
+
+  if (lc($uri_protocol) eq "file") {
+    # expect file path
+    ($extract_file, $extract_range_specifier) = split(/[#,?]/,$specs);
+
+    # decode the file name. URIs may have file/path names with non-alphanumeric characters, which will be encoded with %.  We need to decode these.
+    $decoded_extract_file = uri_unescape($extract_file);
+
+    # verify file exists:
+    if (! -e $decoded_extract_file) {
+      print(STDERR "Error: can't find file: $decoded_extract_file\n"); $error++;
+      next;
+    }
+
+    # use the output_path is specified, otherwise use current working dir.
+    if ($output_path ne "") {
+      $output_file = File::Spec->catfile($output_path, basename($decoded_extract_file));
+    } else {
+      $output_file = basename($decoded_extract_file);
+    }
+
+  } elsif ( lc($uri_protocol) eq "memory") {
+    # expect memory specifier
+    ($extract_pid, $extract_range_specifier) = split(/[#,?]/,$specs);
+
+    # verify pid is currently running
+    $pid_running = kill 0, $extract_pid;
+    if (! $pid_running) {
+      print(STDERR "Error: PID: $extract_pid is NOT running\n"); $error++;
+      next;
+    }
+
+    # get pid filename:
+    $extract_file = "/proc/$extract_pid/mem";
+   
+    # verify file exists:
+    if (! -e $extract_file) {
+      print(STDERR "Error: can't find file: $extract_file\n"); $error++;
+      next;
+    }
+
+    # for extracting from a pid, make the output file in the current dir/path with the pid value as a name.
+    $output_file = "pid${extract_pid}";
+
+    # need to set $decoded_extract_file, because later we use this for other checks.
+    $decoded_extract_file = $extract_file;
+  } else {
+    # error, unrecognized Code Object URI
+    print(STDERR "Error: \'$uri_protocol\' is not recognized as a supported code object URI.\n"); $error++;
+    next;
+  }
+
+  # it is valid to not give a range specifier in a URI, in which case the entire code object will be extracted.
+  if ($extract_range_specifier ne "") {
+    my @tokens;
+    my $str;
+    my $value;
+    my $size_specified = 0;
+
+    @tokens = split(/[&]/,$extract_range_specifier);
+    foreach (@tokens) {
+      ($str,$value) = split(/=/,$_);
+      if ($str eq "size") {
+        $extract_size=$value;
+        $size_specified = 1;
+      } elsif ($str eq "offset") {
+        $extract_offset=$value;
+      }
+    }
+
+    if ($size_specified != 1) {
+      # "size" not specified.  default to rest of file (total size - offset)
+      $extract_size = -s $decoded_extract_file;
+      $extract_size -= $extract_offset;
+    }
+
+  } else {
+    # Error if URI is a memory request, and we have no range_specifier.
+    if ($pid_running) {
+      print(STDERR "Error: must specify a Range Specifier (offset and size) for a memory URI: $uri_str\n"); $error++;
+      next;
+    }
+
+    $extract_offset = 0;
+    $extract_size = -s $decoded_extract_file;
+  }
+
+  # We should have at least a valid size to extract; ignore cases with size=0.
+  if ($extract_size != 0) {
+    print("Reading input file \"$extract_file\" ...\n") if ($verbose);
+
+    # only if this is a File URI.
+    if (lc($uri_protocol) eq "file") {
+      # verify that offset+size does not exceed file size:
+      my $file_size = -s $decoded_extract_file;
+      my $size = int($extract_offset) + int($extract_size);
+      if ( $size > $file_size ) {
+        print(STDERR "Error: requested offset($extract_offset) + size($extract_size) exceeds file size($file_size) for file \"$decoded_extract_file\".\n"); $error++;
+        next;
+      }
+    }
+
+    open(INPUT_FP, "<", $decoded_extract_file) || die $!;
+    binmode INPUT_FP;
+
+    # extract the code object
+    my $co_filename;
+    if (!$output_to_stdout) {
+      $co_filename = "of=\'${output_file}-offset${extract_offset}-size${extract_size}.co\'";
+    }
+
+    my $dd_cmd_str = "$dd_cmd if=\'$decoded_extract_file\' $co_filename skip=$extract_offset count=$extract_size bs=1 status=none";
+
+    print("DD Command: $dd_cmd_str\n") if ($verbose);
+
+    my $dd_ret = system($dd_cmd_str);
+    if ($dd_ret != 0) {
+       print(STDERR "Error: DD command ($dd_cmd_str)  failed with RC: $dd_ret\n"); $error++;
+    }
+
+    print("Extract request:  file: $extract_file offset: $extract_offset size: $extract_size\n") if ($verbose);
+  } else {
+    print("Warning: trying to extract from $extract_file at offset=$extract_offset with size=0.  Nothing to extract.\n") if ($verbose);
+  }
+
+} # end of for each (URI) argument
+
+exit($error);

projects/clr/hipamd/bin/roc-obj-extract.bat

@@ -0,0 +1,2 @@
+@set ROC_OBJ_EXTRACT="%~dp0roc-obj-extract"
+@perl %ROC_OBJ_EXTRACT% %*

projects/clr/hipamd/bin/roc-obj-ls

@@ -0,0 +1,193 @@
+#!/usr/bin/perl
+# Copyright (c) 2020 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+use strict;
+use File::Copy;
+use File::Spec;
+use File::Which;
+use Cwd 'realpath';
+use Getopt::Std;
+use List::Util qw(max);
+use URI::Escape;
+
+sub usage {
+  print("Usage: $0 [-v|h] executable...\n");
+  print("List the URIs of the code objects embedded in the specfied host executables.\n");
+  print("-v \tVerbose output (includes Entry ID)\n");
+  print("-h \tShow this help message\n");
+  exit;
+}
+
+# sub to read a qword. 1st arg is a FP, 2nd arg is ref to destination var.
+sub readq {
+ my ($input_fp, $qword) = @_;
+ read($input_fp, my $bytes, 8) == 8 or die("Error: Failed to read 8 bytes\n");
+ ${$qword} = unpack("Q<", $bytes);
+}
+
+# sub to move address to next alignment boundary
+#  first arg is address to move
+#  second arg is alignment requirement/boundary
+sub align_up {
+	my ($address, $alignment) = @_;
+	$address = int(($address + ($alignment - 1)) / $alignment) * $alignment;
+}
+
+# Process options
+my %options=();
+getopts('vhd', \%options);
+
+if (defined $options{h}) {
+  usage();
+}
+
+# this tool has been deprecated
+print(STDERR "Warning:  This tool has been DEPRECATED. Similar functionality is provided by llvm-objdump in the rocm-llvm package.\n");
+
+my $verbose = $options{v};
+my $debug = $options{d};
+
+my $num_bundles = 1;
+my $bundle_alignment = 4096;
+
+# look for objdump
+my $objdump = which("objdump");
+(-f $objdump) || die("Error: Can't find objdump command\n");
+
+# for each argument (which should be an executable):
+foreach my $executable_file(@ARGV) {
+
+  # debug message
+  print("Reading input file \"$executable_file\" ...\n") if ($debug);
+
+  # verify/open file specified.
+  open (INPUT_FP, "<", $executable_file) || die("Error: failed to open file: $executable_file\n");
+  binmode INPUT_FP;
+
+  # kernel section information
+  my $escaped_name=quotemeta($executable_file);
+  my $bundle_section_name = ".hip_fat";
+  my $bundle_section_size = hex(`$objdump -h $escaped_name | grep $bundle_section_name | awk '{print \$3}'`);
+  my $bundle_section_offset =  hex(`$objdump -h $escaped_name | grep $bundle_section_name | awk '{print \$6}'`);
+
+  $bundle_section_size or die("Error: No kernel section found\n");
+
+  my $bundle_section_end = $bundle_section_offset + $bundle_section_size;
+
+  if ($debug) {
+    printf("Code Objects Bundle section size: %x\n",$bundle_section_size);
+    printf("Code Objects Bundle section offset: %x\n",$bundle_section_offset);
+    printf("Code Objects Bundle section end: %x\n\n",$bundle_section_end);
+  }
+
+  my $current_bundle_offset = $bundle_section_offset;
+  printf("Current Bundle offset: 0x%X\n",$current_bundle_offset) if ($debug);
+
+  # move fp to current_bundle_offset.
+  seek(INPUT_FP, $current_bundle_offset, 0);
+
+  while ($current_bundle_offset < $bundle_section_end) {
+
+    # skip OFFLOAD_BUNDLER_MAGIC_STR
+    my $magic_str;
+    my $read_bytes = read(INPUT_FP, $magic_str, 24);
+    if (($read_bytes != 24) || ($magic_str ne "__CLANG_OFFLOAD_BUNDLE__")) {
+      print(STDERR "Error: Offload bundle magic string not detected\n") if ($debug);
+      last;
+    }
+
+    # read number of bundle entries, which are code objects.
+    my $num_codeobjects;
+    readq(\*INPUT_FP,\$num_codeobjects);
+
+    # header with current bundle number and number of embedded code objcts in that bundle.
+    #    print("Bundle Number: $num_bundles with $num_codeobjects Code Objects:\n") if ($very_verbose);
+
+    my $end_of_current_bundle = $current_bundle_offset;
+
+    # Column Header.
+    printf("%-8s%-40s%35s\n","Bundle#","Entry ID:","URI:") if ($verbose);
+
+    # for each Bundle entry (code object)  ....
+    for (my $iter = 0; $iter < $num_codeobjects; $iter++) {
+
+      print("\nEntry #$iter\n") if $debug;
+
+      # read bundle entry (code object) offset
+      my $entry_offset;
+      my $abs_offset;
+      readq(*INPUT_FP,\$entry_offset);
+      printf("entry_offset: 0x%X\n",$entry_offset) if $debug;
+
+      # read bundle entry (code object) size
+      my $entry_size;
+      readq(*INPUT_FP,\$entry_size);
+      printf("entry_size: 0x%X\n",$entry_size) if $debug;
+
+      # read triple size
+      my $triple_size;
+      readq(*INPUT_FP,\$triple_size);
+      printf("triple_size: 0x%X\n",$triple_size) if $debug;
+
+      # read triple string
+      my $triple;
+      my $read_bytes = read(INPUT_FP, $triple, $triple_size);
+      $read_bytes == $triple_size or die("Error: Fail to parse triple\n");
+      print("triple: $triple\n") if $debug;
+
+      # because the bundle entry's offset is relative to the beginning of the bundled code object section.
+      $abs_offset = int($current_bundle_offset + $entry_offset); 
+
+      # and we need to keep track of where we are in the current bundle.
+      $end_of_current_bundle = int($abs_offset + $entry_size);
+
+      printf("abs_offset: 0x%X\n",$abs_offset) if $debug;
+
+      my $encoded_executable_file = uri_unescape($executable_file);
+
+      printf("%-8s%-40s%35s%s%s%s%s%s%s\n",$num_bundles,$triple,"file:\/\/",$encoded_executable_file,"\#offset=",$abs_offset, "\&size=",$entry_size);
+
+      printf("end_of_current_bundle: 0x%X\n",$end_of_current_bundle) if $debug;
+      printf("Hex values: file:\/\/$encoded_executable_file#offset=0x%X$abs_offset\&size=0x%X\n", $abs_offset, $entry_size) if $debug;
+
+    } # End of for each Bundle entry (code object) ...
+
+    printf("\n") if ($verbose);
+
+    # we've finished listing this current bundle ...
+    printf("current_bundle_offset: %x \n",$current_bundle_offset) if ($debug);
+    printf("bundle_section_end: %x \n", $bundle_section_end) if ($debug);
+
+    # move current_bundle_offset to next alignment boundary.
+    $current_bundle_offset = align_up($end_of_current_bundle,$bundle_alignment);
+    printf("Adjusting for alignment of next bundle: current_bundle_offset: %x \n\n\n", $current_bundle_offset) if ($debug);
+
+    # seek to the end of the current bundle:
+    seek(INPUT_FP, $current_bundle_offset, 0);
+
+    # increment the number of bundles listed.
+    $num_bundles = $num_bundles+1;
+
+  }  # End of while loop
+
+} # End of for each command line argument
+
+exit(0);

projects/clr/hipamd/bin/roc-obj-ls.bat

@@ -0,0 +1,2 @@
+@set ROC_OBJ_LS="%~dp0roc-obj-ls"
+@perl %ROC_OBJ_LS% %*

projects/clr/hipamd/hip-config-amd.cmake.in

@@ -0,0 +1,215 @@
+# Copyright (c) 2023 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# Number of parallel jobs by default is 1
+if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS)
+  set(HIP_CLANG_NUM_PARALLEL_JOBS 1)
+endif()
+
+# Windows Specific Definition here:
+if(WIN32)
+  if(DEFINED ENV{HIP_PATH})
+    file(TO_CMAKE_PATH "$ENV{HIP_PATH}" HIP_PATH)
+  elseif(DEFINED ENV{HIP_DIR})
+    file(TO_CMAKE_PATH "$ENV{HIP_DIR}" HIP_DIR)
+  else()
+    # using the HIP found
+    set(HIP_PATH ${PACKAGE_PREFIX_DIR})
+  endif()
+else()
+  # Linux
+  # If HIP is not installed under ROCm, need this to find HSA assuming HSA is under ROCm
+  if(DEFINED ENV{ROCM_PATH})
+    set(ROCM_PATH "$ENV{ROCM_PATH}")
+  endif()
+
+  # set a default path for ROCM_PATH
+  if(NOT DEFINED ROCM_PATH)
+    set(ROCM_PATH ${PACKAGE_PREFIX_DIR})
+  endif()
+
+endif()
+
+if(WIN32)
+  # Using SDK folder
+  file(TO_CMAKE_PATH "${HIP_PATH}" HIP_CLANG_ROOT)
+  if (NOT EXISTS "${HIP_CLANG_ROOT}/bin/clang.exe")
+    # if using install folder
+    file(TO_CMAKE_PATH "${HIP_PATH}/../lc" HIP_CLANG_ROOT)
+  endif()
+else()
+  set(HIP_CLANG_ROOT "${ROCM_PATH}/llvm")
+endif()
+
+if(NOT HIP_CXX_COMPILER)
+  set(HIP_CXX_COMPILER ${CMAKE_CXX_COMPILER})
+endif()
+
+if(NOT WIN32)
+  find_dependency(AMDDeviceLibs HINTS ${ROCM_PATH} PATHS "/opt/rocm")
+endif()
+
+if(DEFINED AMDGPU_TARGETS AND NOT DEFINED GPU_TARGETS)
+  message(AUTHOR_WARNING "AMDGPU_TARGETS is deprecated. Please use GPU_TARGETS instead.")
+
+  # Set GPU_TARGETS to the value of AMDGPU_TARGETS
+  set(GPU_TARGETS "${AMDGPU_TARGETS}")
+endif()
+
+# If GPU_TARGETS is not defined by the app, amdgpu-arch is run to find the gpu archs
+# of all the devices present in the machine
+if(NOT GPU_TARGETS)
+  if(@BUILD_SHARED_LIBS@)
+    if (WIN32)
+      set(AMDGPU_ARCH "${HIP_CLANG_ROOT}/bin/amdgpu-arch.exe")
+    else()
+      set(AMDGPU_ARCH "${HIP_CLANG_ROOT}/bin/amdgpu-arch")
+    endif()
+  else()
+    set(AMDGPU_ARCH "${ROCM_PATH}/bin/rocm_agent_enumerator")
+  endif()
+
+  execute_process(
+    COMMAND ${AMDGPU_ARCH}
+    RESULT_VARIABLE AMDGPU_ARCH_RESULT
+    OUTPUT_VARIABLE AMDGPU_ARCH_OUTPUT
+    ERROR_VARIABLE  AMDGPU_ARCH_ERROR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_STRIP_TRAILING_WHITESPACE)
+
+  if(AMDGPU_ARCH_ERROR)
+    message(AUTHOR_WARNING
+        " GPU_TARGETS was not set, and system GPU detection was unsuccsessful.\n \n"
+        " The amdgpu-arch tool failed:\n"
+        " Error: '${AMDGPU_ARCH_ERROR}'\n"
+        " Output: '${AMDGPU_ARCH_OUTPUT}'\n \n"
+
+        " As a result, --offload-arch will not be set for subsequent\n"
+        " compilations, and the default architecture\n"
+        " (gfx906 for dynamic build / gfx942 for static build) will be used\n"
+        " for compiling device code in C++ language mode\n")
+  else()
+    # rocm_agent_enumerator adds gfx000 entry
+    string(REPLACE "gfx000\n" "" AMDGPU_ARCH_OUTPUT "${AMDGPU_ARCH_OUTPUT}")
+    if (NOT AMDGPU_ARCH_OUTPUT STREQUAL "")
+      string(REPLACE "\n" ";" AMDGPU_ARCH_OUTPUT ${AMDGPU_ARCH_OUTPUT})
+      set(GPU_TARGETS ${AMDGPU_ARCH_OUTPUT} CACHE STRING "AMD GPU targets to compile for")
+    endif()
+  endif()
+endif()
+
+if (NOT GPU_TARGETS AND NOT @BUILD_SHARED_LIBS@)
+  # The default architecture is gfx942 for static build
+  set(GPU_TARGETS "gfx942" CACHE STRING "AMD GPU targets to compile for")
+endif()
+
+set(GPU_BUILD_TARGETS "${GPU_TARGETS}" CACHE STRING "GPU targets to compile for")
+if(NOT WIN32)
+  find_dependency(amd_comgr HINTS ${ROCM_PATH} PATHS "/opt/rocm")
+endif()
+
+include( "${CMAKE_CURRENT_LIST_DIR}/hip-targets.cmake" )
+
+#Using find_dependency to locate the dependency for the packages
+#This makes the cmake generated file xxxx-targets to supply the linker libraries
+# without worrying other transitive dependencies
+if(NOT WIN32)
+  find_dependency(hsa-runtime64 HINTS ${ROCM_PATH} PATHS "/opt/rocm")
+  find_dependency(Threads)
+endif()
+
+set(_IMPORT_PREFIX ${HIP_PACKAGE_PREFIX_DIR})
+# Right now this is only supported for amd platforms
+set_target_properties(hip::host PROPERTIES
+  INTERFACE_COMPILE_DEFINITIONS "__HIP_PLATFORM_AMD__=1"
+)
+
+set_target_properties(hip::amdhip64 PROPERTIES
+  INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include"
+  INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include"
+)
+
+if(NOT WIN32)
+  set_target_properties(hip::device PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include"
+    INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include"
+  )
+endif()
+
+get_property(compilePropIsSet TARGET hip::device PROPERTY INTERFACE_COMPILE_OPTIONS SET)
+
+if (NOT compilePropIsSet)
+  hip_add_interface_compile_flags(hip::device -x hip)
+endif()
+
+hip_add_interface_link_flags(hip::device --hip-link)
+
+foreach(GPU_TARGET ${GPU_BUILD_TARGETS})
+    if (NOT compilePropIsSet)
+      hip_add_interface_compile_flags(hip::device --offload-arch=${GPU_TARGET})
+    endif()
+    hip_add_interface_link_flags(hip::device --offload-arch=${GPU_TARGET})
+endforeach()
+#Add support for parallel build and link
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+  check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS)
+endif()
+if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1)
+  if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS} )
+    if (NOT compilePropIsSet)
+      hip_add_interface_compile_flags(hip::device -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS} -Wno-format-nonliteral)
+    endif()
+    hip_add_interface_link_flags(hip::device -parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS})
+  else()
+    message(AUTHOR_WARNING "clang compiler doesn't support parallel jobs")
+  endif()
+endif()
+
+# Use HIP_CXX option -print-libgcc-file-name --rtlib=compiler-rt
+# To fetch the compiler rt library file name.
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E env HIPCC_VERBOSE=0
+  ${HIP_CXX_COMPILER} -print-libgcc-file-name --rtlib=compiler-rt
+  OUTPUT_VARIABLE CLANGRT_BUILTINS
+  ERROR_VARIABLE  CLANGRT_Error
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+  ERROR_STRIP_TRAILING_WHITESPACE
+  RESULT_VARIABLE CLANGRT_BUILTINS_FETCH_EXIT_CODE)
+
+if( CLANGRT_Error )
+  if (HIP_CXX_COMPILER MATCHES ".*clang\\+\\+")
+    message(AUTHOR_WARNING "${HIP_CXX_COMPILER}: CLANGRT compiler options not supported.")
+  endif()
+else()
+  # Add support for __fp16 and _Float16, explicitly link with compiler-rt
+  if( "${CLANGRT_BUILTINS_FETCH_EXIT_CODE}" STREQUAL "0" )
+    # The HIP_CXX_COMPILER by default prefers backward slashes for path seperators on windows.
+    # Prefer forward slashes here to avoid escaping issues on certain build systems.
+    if(WIN32)
+      string(REPLACE "\\" "/" CLANGRT_BUILTINS ${CLANGRT_BUILTINS})
+    endif()
+
+    # CLANG_RT Builtins found Successfully Set interface link libraries property
+    set_property(TARGET hip::host APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${CLANGRT_BUILTINS}")
+    set_property(TARGET hip::device APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${CLANGRT_BUILTINS}")
+  else()
+    message(AUTHOR_WARNING "clangrt builtins lib not found: ${CLANGRT_BUILTINS_FETCH_EXIT_CODE}")
+  endif() # CLANGRT_BUILTINS_FETCH_EXIT_CODE Check
+endif() # CLANGRT_Error Check

projects/clr/hipamd/hip-config-nvidia.cmake

@@ -0,0 +1,23 @@
+# Copyright (c) 2023 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+add_library(hip::device INTERFACE IMPORTED)
+add_library(hip::host INTERFACE IMPORTED)
+add_library(hip::amdhip64 INTERFACE IMPORTED)

projects/clr/hipamd/hip-config.cmake.in

@@ -0,0 +1,135 @@
+# Copyright (c) 2020 - 2021 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+@PACKAGE_INIT@
+include(CheckCXXCompilerFlag)
+include(CMakeFindDependencyMacro OPTIONAL RESULT_VARIABLE _CMakeFindDependencyMacro_FOUND)
+if (NOT _CMakeFindDependencyMacro_FOUND)
+  macro(find_dependency dep)
+    if (NOT ${dep}_FOUND)
+      set(cmake_fd_version)
+      if (${ARGC} GREATER 1)
+        set(cmake_fd_version ${ARGV1})
+      endif()
+      set(cmake_fd_exact_arg)
+      if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION_EXACT)
+        set(cmake_fd_exact_arg EXACT)
+      endif()
+      set(cmake_fd_quiet_arg)
+      if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
+        set(cmake_fd_quiet_arg QUIET)
+      endif()
+      set(cmake_fd_required_arg)
+      if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
+        set(cmake_fd_required_arg REQUIRED)
+      endif()
+      find_package(${dep} ${cmake_fd_version}
+          ${cmake_fd_exact_arg}
+          ${cmake_fd_quiet_arg}
+          ${cmake_fd_required_arg}
+      )
+      string(TOUPPER ${dep} cmake_dep_upper)
+      if (NOT ${dep}_FOUND AND NOT ${cmake_dep_upper}_FOUND)
+        set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "${CMAKE_FIND_PACKAGE_NAME} could not be found because dependency ${dep} could not be found.")
+        set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False)
+        return()
+      endif()
+      set(cmake_fd_version)
+      set(cmake_fd_required_arg)
+      set(cmake_fd_quiet_arg)
+      set(cmake_fd_exact_arg)
+    endif()
+  endmacro()
+endif()
+
+set(_HIP_SHELL "SHELL:")
+if(CMAKE_VERSION VERSION_LESS 3.12)
+  set(_HIP_SHELL "")
+endif()
+
+function(hip_add_interface_compile_flags TARGET)
+  set_property(TARGET ${TARGET} APPEND PROPERTY
+    INTERFACE_COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:CXX>:${_HIP_SHELL}${ARGN}>"
+  )
+endfunction()
+
+function(hip_add_interface_link_flags TARGET)
+  if(CMAKE_VERSION VERSION_LESS 3.20)
+    set_property(TARGET ${TARGET} APPEND PROPERTY
+      INTERFACE_LINK_LIBRARIES "${ARGN}"
+    )
+  else()
+    set_property(TARGET ${TARGET} APPEND PROPERTY
+      INTERFACE_LINK_LIBRARIES "$<$<LINK_LANGUAGE:CXX>:${ARGN}>"
+    )
+  endif()
+endfunction()
+
+set(HIP_PACKAGE_PREFIX_DIR  ${PACKAGE_PREFIX_DIR})
+
+set_and_check( hip_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@" )
+set_and_check( hip_INCLUDE_DIRS "${hip_INCLUDE_DIR}" )
+set_and_check( hip_LIB_INSTALL_DIR "@PACKAGE_LIB_INSTALL_DIR@" )
+set_and_check( hip_BIN_INSTALL_DIR "@PACKAGE_BIN_INSTALL_DIR@" )
+if("@HIP_INSTALLS_HIPCC@")
+  if (WIN32)
+    set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc.exe")
+    set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig.exe")
+  else()
+    set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc")
+    set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig")
+  endif()
+endif()
+
+if(NOT DEFINED HIP_PLATFORM)
+  if(NOT DEFINED ENV{HIP_PLATFORM})
+    execute_process(COMMAND ${hip_HIPCONFIG_EXECUTABLE} --platform
+      OUTPUT_VARIABLE HIP_PLATFORM
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+  else()
+    set(HIP_PLATFORM $ENV{HIP_PLATFORM} CACHE STRING "HIP Platform")
+  endif()
+endif()
+
+if(HIP_PLATFORM STREQUAL "amd")
+  set(HIP_RUNTIME "rocclr")
+  set(HIP_COMPILER "clang")
+  include( "${hip_LIB_INSTALL_DIR}/cmake/hip/hip-config-amd.cmake" )
+elseif(HIP_PLATFORM STREQUAL "nvidia")
+  set(HIP_RUNTIME "cuda")
+  set(HIP_COMPILER "nvcc")
+  include( "${hip_LIB_INSTALL_DIR}/cmake/hip/hip-config-nvidia.cmake" )
+else()
+  message(FATAL_ERROR "Unexpected HIP_PLATFORM: " ${HIP_PLATFORM})
+endif()
+
+set( hip_LIBRARIES hip::host hip::device)
+set( hip_LIBRARY ${hip_LIBRARIES})
+
+set(HIP_INCLUDE_DIR ${hip_INCLUDE_DIR})
+set(HIP_INCLUDE_DIRS ${hip_INCLUDE_DIRS})
+set(HIP_LIB_INSTALL_DIR ${hip_LIB_INSTALL_DIR})
+set(HIP_BIN_INSTALL_DIR ${hip_BIN_INSTALL_DIR})
+set(HIP_LIBRARIES ${hip_LIBRARIES})
+set(HIP_LIBRARY ${hip_LIBRARY})
+if("@HIP_INSTALLS_HIPCC@")
+  set(HIP_HIPCC_EXECUTABLE ${hip_HIPCC_EXECUTABLE})
+  set(HIP_HIPCONFIG_EXECUTABLE ${hip_HIPCONFIG_EXECUTABLE})
+endif()

projects/clr/hipamd/include/hip/amd_detail/amd_channel_descriptor.h

@@ -0,0 +1,358 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_common.h>
+#include <hip/driver_types.h>
+#include <hip/amd_detail/amd_hip_vector_types.h>
+#endif
+
+#ifdef __cplusplus
+
+extern "C" HIP_PUBLIC_API
+hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f);
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf1() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf2() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindFloat);
+}
+
+static inline hipChannelFormatDesc hipCreateChannelDescHalf4() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindFloat);
+}
+
+template <typename T>
+static inline hipChannelFormatDesc hipCreateChannelDesc() {
+    return hipCreateChannelDesc(0, 0, 0, 0, hipChannelFormatKindNone);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char>() {
+    int e = (int)sizeof(char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed char>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned char>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar1>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char1>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar2>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char2>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__  // vector3 is the same as vector4
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar3>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char3>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uchar4>() {
+    int e = (int)sizeof(unsigned char) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<char4>() {
+    int e = (int)sizeof(signed char) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned short>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed short>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort1>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short1>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort2>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short2>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort3>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short3>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ushort4>() {
+    int e = (int)sizeof(unsigned short) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<short4>() {
+    int e = (int)sizeof(signed short) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned int>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed int>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint1>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int1>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint2>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int2>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint3>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int3>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<uint4>() {
+    int e = (int)sizeof(unsigned int) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<int4>() {
+    int e = (int)sizeof(signed int) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float1>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float2>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindFloat);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float3>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindFloat);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<float4>() {
+    int e = (int)sizeof(float) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindFloat);
+}
+
+#if !defined(__LP64__)
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<unsigned long>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<signed long>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong1>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long1>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong2>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long2>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
+}
+
+#ifndef __GNUC__
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong3>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long3>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
+}
+#endif
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<ulong4>() {
+    int e = (int)sizeof(unsigned long) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
+}
+
+template <>
+inline hipChannelFormatDesc hipCreateChannelDesc<long4>() {
+    int e = (int)sizeof(signed long) * 8;
+    return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
+}
+#endif /* !__LP64__ */
+
+#else
+
+struct hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w,
+                                                 enum hipChannelFormatKind f);
+
+#endif /* __cplusplus */
+
+#endif /* !HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H */

projects/clr/hipamd/include/hip/amd_detail/amd_hip_atomic.h

@@ -0,0 +1,857 @@
+/*
+Copyright (c) 2015 - Present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include "amd_device_functions.h"
+#endif
+
+template<bool B, typename T, typename F> struct Cond_t;
+
+template<typename T, typename F> struct Cond_t<true, T, F> { using type = T; };
+template<typename T, typename F> struct Cond_t<false, T, F> { using type = F; };
+
+#if !__HIP_DEVICE_COMPILE__
+//TODO: Remove this after compiler pre-defines the following Macros.
+#define __HIP_MEMORY_SCOPE_SINGLETHREAD 1
+#define __HIP_MEMORY_SCOPE_WAVEFRONT 2
+#define __HIP_MEMORY_SCOPE_WORKGROUP 3
+#define __HIP_MEMORY_SCOPE_AGENT 4
+#define __HIP_MEMORY_SCOPE_SYSTEM 5
+#endif
+
+#if !defined(__HIPCC_RTC__)
+#include "amd_hip_unsafe_atomics.h"
+#endif
+
+// Atomic expanders
+template<
+  int mem_order = __ATOMIC_SEQ_CST,
+  int mem_scope= __HIP_MEMORY_SCOPE_SYSTEM,
+  typename T,
+  typename Op,
+  typename F>
+inline
+__attribute__((always_inline, device))
+T hip_cas_expander(T* p, T x, Op op, F f) noexcept
+{
+  using FP = __attribute__((address_space(0))) const void*;
+
+  __device__
+  extern bool is_shared_workaround(FP) asm("llvm.amdgcn.is.shared");
+
+  if (is_shared_workaround((FP)p))
+    return f();
+
+  using U = typename Cond_t<
+    sizeof(T) == sizeof(unsigned int), unsigned int, unsigned long long>::type;
+
+  auto q = reinterpret_cast<U*>(p);
+
+  U tmp0{__hip_atomic_load(q, mem_order, mem_scope)};
+  U tmp1;
+  do {
+    tmp1 = tmp0;
+
+    op(reinterpret_cast<T&>(tmp1), x);
+  } while (!__hip_atomic_compare_exchange_strong(q, &tmp0, tmp1, mem_order,
+                                                 mem_order, mem_scope));
+
+  return reinterpret_cast<const T&>(tmp0);
+}
+
+template<
+  int mem_order = __ATOMIC_SEQ_CST,
+  int mem_scope= __HIP_MEMORY_SCOPE_SYSTEM,
+  typename T,
+  typename Cmp,
+  typename F>
+inline
+__attribute__((always_inline, device))
+T hip_cas_extrema_expander(T* p, T x, Cmp cmp, F f) noexcept
+{
+  using FP = __attribute__((address_space(0))) const void*;
+
+  __device__
+  extern bool is_shared_workaround(FP) asm("llvm.amdgcn.is.shared");
+
+  if (is_shared_workaround((FP)p))
+    return f();
+
+  using U = typename Cond_t<
+    sizeof(T) == sizeof(unsigned int), unsigned int, unsigned long long>::type;
+
+  auto q = reinterpret_cast<U*>(p);
+
+  U tmp{__hip_atomic_load(q, mem_order, mem_scope)};
+  while (cmp(x, reinterpret_cast<const T&>(tmp)) &&
+         !__hip_atomic_compare_exchange_strong(q, &tmp, x, mem_order, mem_order,
+                                               mem_scope));
+
+  return reinterpret_cast<const T&>(tmp);
+}
+
+__device__
+inline
+unsigned short int atomicCAS(unsigned short int* address, unsigned short int compare,
+                              unsigned short int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned short int atomicCAS_system(unsigned short int* address, unsigned short int compare,
+                              unsigned short int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+int atomicCAS(int* address, int compare, int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+    return compare;
+}
+
+__device__
+inline
+int atomicCAS_system(int* address, int compare, int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+    return compare;
+}
+
+__device__
+inline
+unsigned int atomicCAS(unsigned int* address, unsigned int compare, unsigned int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned int atomicCAS_system(unsigned int* address, unsigned int compare, unsigned int val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+unsigned long atomicCAS(unsigned long* address, unsigned long compare, unsigned long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned long atomicCAS_system(unsigned long* address, unsigned long compare, unsigned long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+unsigned long long atomicCAS(unsigned long long* address, unsigned long long compare,
+                             unsigned long long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+  return compare;
+}
+
+__device__
+inline
+unsigned long long atomicCAS_system(unsigned long long* address, unsigned long long compare,
+                                    unsigned long long val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+  return compare;
+}
+
+__device__
+inline
+float atomicCAS(float* address, float compare, float val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+    return compare;
+}
+
+__device__
+inline
+float atomicCAS_system(float* address, float compare, float val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+    return compare;
+}
+
+__device__
+inline
+double atomicCAS(double* address, double compare, double val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_AGENT);
+    return compare;
+}
+
+__device__
+inline
+double atomicCAS_system(double* address, double compare, double val) {
+  __hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
+                                       __HIP_MEMORY_SCOPE_SYSTEM);
+    return compare;
+}
+
+__device__
+inline
+int atomicAdd(int* address, int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicAdd_system(int* address, int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicAdd(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicAdd_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicAdd(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicAdd_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicAdd(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicAdd_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+#if defined(__has_extension) && __has_extension(clang_atomic_attributes)
+#define __HIP_FINE_GRAINED_MEMORY [[clang::atomic(fine_grained_memory)]]
+#else
+#define __HIP_FINE_GRAINED_MEMORY
+#endif
+
+__device__
+inline
+float atomicAdd(float* address, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {  
+    return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicAdd_system(float* address, float val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+#if !defined(__HIPCC_RTC__)
+HIP_DEPRECATED("use atomicAdd instead")
+#endif // !defined(__HIPCC_RTC__)
+__device__
+inline
+void atomicAddNoRet(float* address, float val)
+{
+  unsafeAtomicAdd(address, val);
+}
+
+__device__
+inline
+double atomicAdd(double* address, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicAdd_system(double* address, double val) {
+  return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicSub(int* address, int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicSub_system(int* address, int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicSub(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicSub_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicSub(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicSub_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicSub(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicSub_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicSub(float* address, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, -val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicSub_system(float* address, float val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+double atomicSub(double* address, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicAdd(address, -val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicSub_system(double* address, double val) {
+  return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicExch(int* address, int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicExch_system(int* address, int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicExch(unsigned int* address, unsigned int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicExch_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicExch(unsigned long* address, unsigned long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicExch_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicExch(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicExch_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicExch(float* address, float val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+float atomicExch_system(float* address, float val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+double atomicExch(double* address, double val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+double atomicExch_system(double* address, double val) {
+  return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicMin(int* address, int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicMin_system(int* address, int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicMin(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicMin_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicMin(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicMin_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicMin(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicMin_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+long long atomicMin(long long* address, long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+long long atomicMin_system(long long* address, long long val) {
+  return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicMin(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicMin_system(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMin(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMin_system(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMin(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_min(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+int atomicMax(int* address, int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicMax_system(int* address, int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicMax(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicMax_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicMax(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicMax_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicMax(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicMax_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+__device__
+inline
+long long atomicMax(long long* address, long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+long long atomicMax_system(long long* address, long long val) {
+  return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+float atomicMax(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+float atomicMax_system(float* addr, float val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMax(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#endif
+}
+
+__device__
+inline
+double atomicMax_system(double* addr, double val) {
+#if defined(__AMDGCN_UNSAFE_FP_ATOMICS__)
+  return unsafeAtomicMax(addr, val);
+#else
+  __HIP_FINE_GRAINED_MEMORY {
+    return __hip_atomic_fetch_max(addr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#endif
+}
+
+__device__
+inline
+unsigned int atomicInc(unsigned int* address, unsigned int val)
+{
+  return __builtin_amdgcn_atomic_inc32(address, val, __ATOMIC_RELAXED, "agent");
+}
+
+__device__
+inline
+unsigned int atomicDec(unsigned int* address, unsigned int val)
+{
+  return __builtin_amdgcn_atomic_dec32(address, val, __ATOMIC_RELAXED, "agent");
+}
+
+__device__
+inline
+int atomicAnd(int* address, int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicAnd_system(int* address, int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicAnd(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicAnd_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicAnd(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicAnd_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicAnd(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicAnd_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicOr(int* address, int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicOr_system(int* address, int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicOr(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicOr_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicOr(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicOr_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicOr(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicOr_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+int atomicXor(int* address, int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+int atomicXor_system(int* address, int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned int atomicXor(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned int atomicXor_system(unsigned int* address, unsigned int val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long atomicXor(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long atomicXor_system(unsigned long* address, unsigned long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}
+
+__device__
+inline
+unsigned long long atomicXor(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+}
+
+__device__
+inline
+unsigned long long atomicXor_system(unsigned long long* address, unsigned long long val) {
+  return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+}

projects/clr/hipamd/include/hip/amd_detail/amd_hip_bfloat16.h

@@ -0,0 +1,293 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2019 - 2022 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*!\file
+ * \brief hip_bfloat16.h provides struct for hip_bfloat16 typedef
+ */
+
+#ifndef _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_BFLOAT16_H_
+#define _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_BFLOAT16_H_
+
+#include "host_defines.h"
+#if defined(__HIPCC_RTC__)
+    #define __HOST_DEVICE__ __device__
+#else
+    #define __HOST_DEVICE__ __host__ __device__
+#endif
+
+#if __cplusplus < 201103L || !defined(__HIPCC__)
+
+// If this is a C compiler, C++ compiler below C++11, or a host-only compiler, we only
+// include a minimal definition of hip_bfloat16
+
+#include <stdint.h>
+/*! \brief Struct to represent a 16 bit brain floating point number. */
+typedef struct
+{
+    uint16_t data;
+} hip_bfloat16;
+
+#else // __cplusplus < 201103L || !defined(__HIPCC__)
+
+#include <hip/hip_runtime.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wshadow"
+struct hip_bfloat16
+{
+    __hip_uint16_t data;
+
+    enum truncate_t
+    {
+        truncate
+    };
+
+    __HOST_DEVICE__ hip_bfloat16() = default;
+
+    // round upper 16 bits of IEEE float to convert to bfloat16
+    explicit __HOST_DEVICE__ hip_bfloat16(float f)
+        : data(float_to_bfloat16(f))
+    {
+    }
+
+    explicit __HOST_DEVICE__ hip_bfloat16(float f, truncate_t)
+        : data(truncate_float_to_bfloat16(f))
+    {
+    }
+
+    // zero extend lower 16 bits of bfloat16 to convert to IEEE float
+    __HOST_DEVICE__ operator float() const
+    {
+        union
+        {
+            __hip_uint32_t int32;
+            float    fp32;
+        } u = {__hip_uint32_t(data) << 16};
+        return u.fp32;
+    }
+
+    __HOST_DEVICE__ hip_bfloat16 &operator=(const float& f)
+    {
+       data = float_to_bfloat16(f);
+       return *this;
+    }
+
+    static  __HOST_DEVICE__ hip_bfloat16 round_to_bfloat16(float f)
+    {
+        hip_bfloat16 output;
+        output.data = float_to_bfloat16(f);
+        return output;
+    }
+
+    static  __HOST_DEVICE__ hip_bfloat16 round_to_bfloat16(float f, truncate_t)
+    {
+        hip_bfloat16 output;
+        output.data = truncate_float_to_bfloat16(f);
+        return output;
+    }
+
+private:
+    static __HOST_DEVICE__ __hip_uint16_t float_to_bfloat16(float f)
+    {
+        union
+        {
+            float    fp32;
+            __hip_uint32_t int32;
+        } u = {f};
+        if(~u.int32 & 0x7f800000)
+        {
+            // When the exponent bits are not all 1s, then the value is zero, normal,
+            // or subnormal. We round the bfloat16 mantissa up by adding 0x7FFF, plus
+            // 1 if the least significant bit of the bfloat16 mantissa is 1 (odd).
+            // This causes the bfloat16's mantissa to be incremented by 1 if the 16
+            // least significant bits of the float mantissa are greater than 0x8000,
+            // or if they are equal to 0x8000 and the least significant bit of the
+            // bfloat16 mantissa is 1 (odd). This causes it to be rounded to even when
+            // the lower 16 bits are exactly 0x8000. If the bfloat16 mantissa already
+            // has the value 0x7f, then incrementing it causes it to become 0x00 and
+            // the exponent is incremented by one, which is the next higher FP value
+            // to the unrounded bfloat16 value. When the bfloat16 value is subnormal
+            // with an exponent of 0x00 and a mantissa of 0x7F, it may be rounded up
+            // to a normal value with an exponent of 0x01 and a mantissa of 0x00.
+            // When the bfloat16 value has an exponent of 0xFE and a mantissa of 0x7F,
+            // incrementing it causes it to become an exponent of 0xFF and a mantissa
+            // of 0x00, which is Inf, the next higher value to the unrounded value.
+            u.int32 += 0x7fff + ((u.int32 >> 16) & 1); // Round to nearest, round to even
+        }
+        else if(u.int32 & 0xffff)
+        {
+            // When all of the exponent bits are 1, the value is Inf or NaN.
+            // Inf is indicated by a zero mantissa. NaN is indicated by any nonzero
+            // mantissa bit. Quiet NaN is indicated by the most significant mantissa
+            // bit being 1. Signaling NaN is indicated by the most significant
+            // mantissa bit being 0 but some other bit(s) being 1. If any of the
+            // lower 16 bits of the mantissa are 1, we set the least significant bit
+            // of the bfloat16 mantissa, in order to preserve signaling NaN in case
+            // the bloat16's mantissa bits are all 0.
+            u.int32 |= 0x10000; // Preserve signaling NaN
+        }
+        return __hip_uint16_t(u.int32 >> 16);
+    }
+
+    // Truncate instead of rounding, preserving SNaN
+    static __HOST_DEVICE__ __hip_uint16_t truncate_float_to_bfloat16(float f)
+    {
+        union
+        {
+            float    fp32;
+            __hip_uint32_t int32;
+        } u = {f};
+        return __hip_uint16_t(u.int32 >> 16) | (!(~u.int32 & 0x7f800000) && (u.int32 & 0xffff));
+    }
+};
+#pragma clang diagnostic pop
+
+typedef struct
+{
+    __hip_uint16_t data;
+} hip_bfloat16_public;
+
+static_assert(__hip_internal::is_standard_layout<hip_bfloat16>{},
+              "hip_bfloat16 is not a standard layout type, and thus is "
+              "incompatible with C.");
+
+static_assert(__hip_internal::is_trivial<hip_bfloat16>{},
+              "hip_bfloat16 is not a trivial type, and thus is "
+              "incompatible with C.");
+#if !defined(__HIPCC_RTC__)
+static_assert(sizeof(hip_bfloat16) == sizeof(hip_bfloat16_public)
+                  && offsetof(hip_bfloat16, data) == offsetof(hip_bfloat16_public, data),
+              "internal hip_bfloat16 does not match public hip_bfloat16");
+
+inline std::ostream& operator<<(std::ostream& os, const hip_bfloat16& bf16)
+{
+  return os << float(bf16);
+}
+#endif
+
+inline __HOST_DEVICE__ hip_bfloat16 operator+(hip_bfloat16 a)
+{
+    return a;
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator-(hip_bfloat16 a)
+{
+    a.data ^= 0x8000;
+    return a;
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator+(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) + float(b));
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator-(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) - float(b));
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator*(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) * float(b));
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator/(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return hip_bfloat16(float(a) / float(b));
+}
+inline __HOST_DEVICE__ bool operator<(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return float(a) < float(b);
+}
+inline __HOST_DEVICE__ bool operator==(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return float(a) == float(b);
+}
+inline __HOST_DEVICE__ bool operator>(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return b < a;
+}
+inline __HOST_DEVICE__ bool operator<=(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return !(a > b);
+}
+inline __HOST_DEVICE__ bool operator!=(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return !(a == b);
+}
+inline __HOST_DEVICE__ bool operator>=(hip_bfloat16 a, hip_bfloat16 b)
+{
+    return !(a < b);
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator+=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a + b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator-=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a - b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator*=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a * b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator/=(hip_bfloat16& a, hip_bfloat16 b)
+{
+    return a = a / b;
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator++(hip_bfloat16& a)
+{
+    return a += hip_bfloat16(1.0f);
+}
+inline __HOST_DEVICE__ hip_bfloat16& operator--(hip_bfloat16& a)
+{
+    return a -= hip_bfloat16(1.0f);
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator++(hip_bfloat16& a, int)
+{
+    hip_bfloat16 orig = a;
+    ++a;
+    return orig;
+}
+inline __HOST_DEVICE__ hip_bfloat16 operator--(hip_bfloat16& a, int)
+{
+    hip_bfloat16 orig = a;
+    --a;
+    return orig;
+}
+
+namespace std
+{
+    constexpr __HOST_DEVICE__ bool isinf(hip_bfloat16 a)
+    {
+        return !(~a.data & 0x7f80) && !(a.data & 0x7f);
+    }
+    constexpr __HOST_DEVICE__ bool isnan(hip_bfloat16 a)
+    {
+        return !(~a.data & 0x7f80) && +(a.data & 0x7f);
+    }
+    constexpr __HOST_DEVICE__ bool iszero(hip_bfloat16 a)
+    {
+        return !(a.data & 0x7fff);
+    }
+}
+
+#endif // __cplusplus < 201103L || !defined(__HIPCC__)
+
+#endif // _HIP_BFLOAT16_H_

projects/clr/hipamd/include/hip/amd_detail/amd_hip_common.h

@@ -0,0 +1,32 @@
+/*
+Copyright (c) 2019 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
+
+#if defined(__clang__) && defined(__HIP__)
+#define __HIP_CLANG_ONLY__ 1
+#else
+#define __HIP_CLANG_ONLY__ 0
+#endif
+
+#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_complex.h

@@ -0,0 +1,174 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/* The header defines complex numbers and related functions*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
+
+#if !defined(__HIPCC_RTC__)
+#include "hip/amd_detail/amd_hip_vector_types.h"
+#endif
+
+#if defined(__HIPCC_RTC__)
+#define __HOST_DEVICE__ __device__
+#else
+#define __HOST_DEVICE__ __host__ __device__
+// TODO: Clang has a bug which allows device functions to call std functions
+// when std functions are introduced into default namespace by using statement.
+// math.h may be included after this bug is fixed.
+#if __cplusplus
+#include <cmath>
+#else
+#include "math.h"
+#endif
+#endif // !defined(__HIPCC_RTC__)
+
+typedef float2 hipFloatComplex;
+
+__HOST_DEVICE__ static inline float hipCrealf(hipFloatComplex z) { return z.x; }
+
+__HOST_DEVICE__ static inline float hipCimagf(hipFloatComplex z) { return z.y; }
+
+__HOST_DEVICE__ static inline hipFloatComplex make_hipFloatComplex(float a, float b) {
+    hipFloatComplex z;
+    z.x = a;
+    z.y = b;
+    return z;
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipConjf(hipFloatComplex z) {
+    hipFloatComplex ret;
+    ret.x = z.x;
+    ret.y = -z.y;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline float hipCsqabsf(hipFloatComplex z) {
+    return z.x * z.x + z.y * z.y;
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCaddf(hipFloatComplex p, hipFloatComplex q) {
+    return make_hipFloatComplex(p.x + q.x, p.y + q.y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCsubf(hipFloatComplex p, hipFloatComplex q) {
+    return make_hipFloatComplex(p.x - q.x, p.y - q.y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCmulf(hipFloatComplex p, hipFloatComplex q) {
+    return make_hipFloatComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipCdivf(hipFloatComplex p, hipFloatComplex q) {
+    float sqabs = hipCsqabsf(q);
+    hipFloatComplex ret;
+    ret.x = (p.x * q.x + p.y * q.y) / sqabs;
+    ret.y = (p.y * q.x - p.x * q.y) / sqabs;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline float hipCabsf(hipFloatComplex z) { return sqrtf(hipCsqabsf(z)); }
+
+
+typedef double2 hipDoubleComplex;
+
+__HOST_DEVICE__ static inline double hipCreal(hipDoubleComplex z) { return z.x; }
+
+__HOST_DEVICE__ static inline double hipCimag(hipDoubleComplex z) { return z.y; }
+
+__HOST_DEVICE__ static inline hipDoubleComplex make_hipDoubleComplex(double a, double b) {
+    hipDoubleComplex z;
+    z.x = a;
+    z.y = b;
+    return z;
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipConj(hipDoubleComplex z) {
+    hipDoubleComplex ret;
+    ret.x = z.x;
+    ret.y = -z.y;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline double hipCsqabs(hipDoubleComplex z) {
+    return z.x * z.x + z.y * z.y;
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCadd(hipDoubleComplex p, hipDoubleComplex q) {
+    return make_hipDoubleComplex(p.x + q.x, p.y + q.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCsub(hipDoubleComplex p, hipDoubleComplex q) {
+    return make_hipDoubleComplex(p.x - q.x, p.y - q.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCmul(hipDoubleComplex p, hipDoubleComplex q) {
+    return make_hipDoubleComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCdiv(hipDoubleComplex p, hipDoubleComplex q) {
+    double sqabs = hipCsqabs(q);
+    hipDoubleComplex ret;
+    ret.x = (p.x * q.x + p.y * q.y) / sqabs;
+    ret.y = (p.y * q.x - p.x * q.y) / sqabs;
+    return ret;
+}
+
+__HOST_DEVICE__ static inline double hipCabs(hipDoubleComplex z) { return sqrt(hipCsqabs(z)); }
+
+typedef hipFloatComplex hipComplex;
+
+__HOST_DEVICE__ static inline hipComplex make_hipComplex(float x, float y) {
+    return make_hipFloatComplex(x, y);
+}
+
+__HOST_DEVICE__ static inline hipFloatComplex hipComplexDoubleToFloat(hipDoubleComplex z) {
+    return make_hipFloatComplex((float)z.x, (float)z.y);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipComplexFloatToDouble(hipFloatComplex z) {
+    return make_hipDoubleComplex((double)z.x, (double)z.y);
+}
+
+__HOST_DEVICE__ static inline hipComplex hipCfmaf(hipComplex p, hipComplex q, hipComplex r) {
+    float real = (p.x * q.x) + r.x;
+    float imag = (q.x * p.y) + r.y;
+
+    real = -(p.y * q.y) + real;
+    imag = (p.x * q.y) + imag;
+
+    return make_hipComplex(real, imag);
+}
+
+__HOST_DEVICE__ static inline hipDoubleComplex hipCfma(hipDoubleComplex p, hipDoubleComplex q,
+                                                           hipDoubleComplex r) {
+    double real = (p.x * q.x) + r.x;
+    double imag = (q.x * p.y) + r.y;
+
+    real = -(p.y * q.y) + real;
+    imag = (p.x * q.y) + imag;
+
+    return make_hipDoubleComplex(real, imag);
+}
+
+#endif //HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp4.h

@@ -0,0 +1,460 @@
+/*
+Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#pragma once
+
+#include "amd_hip_mx_common.h"
+
+#include "amd_hip_fp16.h"
+#include "amd_hip_bf16.h"
+#include "amd_hip_fp8.h"
+
+#include "amd_hip_ocp_types.h"
+#include "amd_hip_ocp_host.hpp"
+
+#if defined(__HIPCC_RTC__)
+#define __FP4_HOST_DEVICE__ __device__
+#define __FP4_HOST_DEVICE_STATIC__ __FP4_HOST_DEVICE__ static
+#else
+#define __FP4_HOST_DEVICE__ __host__ __device__
+#define __FP4_HOST_DEVICE_STATIC__ __FP4_HOST_DEVICE__ static inline
+#endif  // __HIPCC_RTC__
+
+typedef __hip_fp8_storage_t __hip_fp4_storage_t;
+typedef __hip_fp8_storage_t __hip_fp4x2_storage_t;
+typedef __hip_fp8x2_storage_t __hip_fp4x4_storage_t;
+
+static_assert(sizeof(__hip_fp4_storage_t[4]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp4x2_storage_t[4]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp4x4_storage_t[2]) == sizeof(uint32_t));
+
+enum __hip_fp4_interpretation_t {
+  __HIP_E2M1 = 0,
+};
+
+namespace internal {
+__FP4_HOST_DEVICE_STATIC__ __amd_fp16_storage_t half_to_f16(const __half val) {
+  __half_raw tmp = val;
+  return tmp.data;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __amd_fp16x2_storage_t half2_to_f16x2(const __half2 val) {
+  __half2_raw tmp = val;
+  return tmp.data;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __amd_bf16_storage_t hipbf16_to_bf16(const __hip_bfloat16 val) {
+  static_assert(sizeof(__hip_bfloat16) == sizeof(__amd_bf16_storage_t));
+  union {
+    __hip_bfloat16 hip_bf16;
+    __amd_bf16_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __amd_bf16x2_storage_t hipbf162_to_bf16x2(const __hip_bfloat162 val) {
+  static_assert(sizeof(__hip_bfloat162) == sizeof(__amd_bf16x2_storage_t));
+  union {
+    __hip_bfloat162 hip_bf16;
+    __amd_bf16x2_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+}  // namespace internal
+
+// Note: Ignore rounding input on AMD GPUs for now. At the moment AMD GPUs do not support rounding
+// modes, all the inputs are rounded to nearest or use an input to do stochastic rounding.
+// We hide the rounding variable to not trigger the unused variable compiler warning.
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t __hip_cvt_bfloat16raw_to_fp4(
+    const __hip_bfloat16_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_bf16(
+      u.ui32, internal::hipbf162_to_bf16x2(__hip_bfloat162{x, 0}), 1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M1, true>(
+      internal::hipbf16_to_bf16(x), 0 /* scale */);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t __hip_cvt_bfloat16raw2_to_fp4x2(
+    const __hip_bfloat162_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_bf16(u.ui32, internal::hipbf162_to_bf16x2(x),
+                                                     1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  auto bf16x2 = internal::hipbf162_to_bf16x2(x);
+  u.ui32 |=
+      fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M1, true>(bf16x2[1], 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |=
+      fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M1, true>(bf16x2[0], 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t
+__hip_cvt_double_to_fp4(const double x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+                        const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, float(x), 0.0f, 1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M1, true>(float(x), 0 /* scale */);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t __hip_cvt_double2_to_fp4x2(
+    const double2 x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 =
+      __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, float(x.x), float(x.y), 1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(float(x.y), 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(float(x.x), 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t
+__hip_cvt_float_to_fp4(const float x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+                       const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, x, 0.0f, 1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M1, true>(x, 0 /*scale*/);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t
+__hip_cvt_float2_to_fp4x2(const float2 x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+                          const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f32(u.ui32, x.x, x.y, 1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(x.y, 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M1, true>(x.x, 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __half_raw __hip_cvt_fp4_to_halfraw(
+    const __hip_fp4_storage_t x, const __hip_fp4_interpretation_t /* fp4_interpretation */) {
+  __half2_raw ret;
+#if __gfx950__
+  ret.data = __amd_fp16x2_storage_t{__builtin_amdgcn_cvt_scalef32_pk_f16_fp4(x, 0, 0)};
+#else
+  using namespace fcbx;
+  ret.data =
+      __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x & 0xFu, 0),
+                             to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x >> 4, 0)};
+#endif
+  return ret.x;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __half2_raw __hip_cvt_fp4x2_to_halfraw2(
+    const __hip_fp4x2_storage_t x, const __hip_fp4_interpretation_t /* fp4_interpretation */) {
+  __half2_raw ret;
+#if __gfx950__
+  ret.data = __amd_fp16x2_storage_t{__builtin_amdgcn_cvt_scalef32_pk_f16_fp4(x, 0, 0)};
+#else
+  using namespace fcbx;
+  ret.data =
+      __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x & 0xFu, 0),
+                             to_float<__amd_fp16_storage_t, Encoding::E2M1, true>(x >> 4, 0)};
+#endif
+  return ret;
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4_storage_t __hip_cvt_halfraw_to_fp4(
+    const __half_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4_storage_t fp4[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f16(u.ui32, internal::half2_to_f16x2(__half2{x, 0}),
+                                                    1.0f /* scale */, 0);
+  return u.fp4[0];
+#else
+  u.ui32 = fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M1, true>(
+      internal::half_to_f16(x), 0 /* scale */);
+  return u.fp4[0];
+#endif
+}
+
+__FP4_HOST_DEVICE_STATIC__ __hip_fp4x2_storage_t __hip_cvt_halfraw2_to_fp4x2(
+    const __half2_raw x, const __hip_fp4_interpretation_t /* fp4_interpretation */,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp4x2_storage_t fp4x2[4];
+  } u{0};
+#if __gfx950__
+  u.ui32 = __builtin_amdgcn_cvt_scalef32_pk_fp4_f16(u.ui32, internal::half2_to_f16x2(x),
+                                                    1.0f /* scale */, 0);
+  return u.fp4x2[0];
+#else
+  auto fp16x2 = internal::half2_to_f16x2(x);
+  u.ui32 |=
+      fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M1, true>(fp16x2[1], 0 /*scale*/);
+  u.ui32 <<= 4;
+  u.ui32 |=
+      fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M1, true>(fp16x2[0], 0 /*scale*/);
+  return u.fp4x2[0];
+#endif
+}
+
+struct __hip_fp4_e2m1 {
+  __hip_fp4_storage_t __x;
+
+ public:
+  __FP4_HOST_DEVICE__ __hip_fp4_e2m1() = default;
+
+#if !defined(__HIP_NO_FP4_CONVERSIONS__)
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const __half f)
+      : __x(__hip_cvt_halfraw_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const __hip_bfloat16 f)
+      : __x(__hip_cvt_bfloat16raw_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__
+  explicit __hip_fp4_e2m1(const double f)
+      : __x(__hip_cvt_double_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const float f)
+      : __x(__hip_cvt_float_to_fp4(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const long long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const short int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned long long int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4_e2m1(const unsigned short int val)
+      : __x(__hip_cvt_float_to_fp4(float(val), __HIP_E2M1, hipRoundNearest)) {}
+#endif  // #if !defined(__HIP_NO_FP4_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+  __FP4_HOST_DEVICE__ operator __half_raw() const {
+    return __hip_cvt_fp4_to_halfraw(__x, __HIP_E2M1);
+  }
+
+  __FP4_HOST_DEVICE__ operator __hip_bfloat16_raw() const {
+    static_assert(sizeof(__hip_bfloat16_raw[2]) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat16_raw bf16_raw[2];
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    u.bf16x2 = __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x & 0xFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return u.bf16_raw[0];
+  }
+
+  __FP4_HOST_DEVICE__ operator float() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto ret = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    __amd_floatx2_storage_t ret{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                to_float<float, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return ret[0];
+  }
+
+  __FP4_HOST_DEVICE__ operator double() const { return double(float(*this)); }
+#endif  // !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+};
+
+/* FP4x2 E2M1 */
+struct __hip_fp4x2_e2m1 {
+  __hip_fp4x2_storage_t __x;
+
+  __FP4_HOST_DEVICE__ __hip_fp4x2_e2m1() = default;
+
+#if !defined(__HIP_NO_FP4_CONVERSIONS__)
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const __half2 f)
+      : __x(__hip_cvt_halfraw2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const __hip_bfloat162 f)
+      : __x(__hip_cvt_bfloat16raw2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const double2 f)
+      : __x(__hip_cvt_double2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ explicit __hip_fp4x2_e2m1(const float2 f)
+      : __x(__hip_cvt_float2_to_fp4x2(f, __HIP_E2M1, hipRoundNearest)) {}
+
+#endif  // #if !defined(__HIP_NO_FP4_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+  __FP4_HOST_DEVICE__ operator __half2_raw() const {
+    return __hip_cvt_fp4x2_to_halfraw2(__x, __HIP_E2M1);
+  }
+
+  __FP4_HOST_DEVICE__ operator __hip_bfloat162_raw() const {
+    static_assert(sizeof(__hip_bfloat162_raw) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat162_raw bf162_raw;
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    u.bf16x2 = __builtin_amdgcn_cvt_scalef32_pk_bf16_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x & 0xFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return u.bf162_raw;
+  }
+
+  __FP4_HOST_DEVICE__ operator float2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto fp32x2 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                          to_float<float, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return float2(fp32x2[0], fp32x2[1]);
+  }
+
+  __FP4_HOST_DEVICE__ operator double2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto fp32x2 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                          to_float<float, Encoding::E2M1, true>(__x >> 4, 0)};
+#endif
+    return double2(fp32x2[0], fp32x2[1]);
+  }
+#endif  // !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+};
+
+/* FP4x4 E2M1 */
+struct __hip_fp4x4_e2m1 {
+  __hip_fp4x4_storage_t __x;
+
+  __FP4_HOST_DEVICE__ inline __hip_fp4x4_e2m1() = default;
+
+#if !defined(__HIP_NO_FP4_CONVERSIONS__)
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const __half2 low, const __half2 high)
+      : __x(__hip_cvt_halfraw2_to_fp4x2(high, __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_halfraw2_to_fp4x2(low, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const __hip_bfloat162 low,
+                                                       const __hip_bfloat162 high)
+      : __x(__hip_cvt_bfloat16raw2_to_fp4x2(high, __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_bfloat16raw2_to_fp4x2(low, __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const double4 f)
+      : __x(__hip_cvt_double2_to_fp4x2(double2(f.z, f.w), __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_double2_to_fp4x2(double2(f.x, f.y), __HIP_E2M1, hipRoundNearest)) {}
+
+  __FP4_HOST_DEVICE__ inline explicit __hip_fp4x4_e2m1(const float4 f)
+      : __x(__hip_cvt_float2_to_fp4x2(float2(f.z, f.w), __HIP_E2M1, hipRoundNearest) << 8 |
+            __hip_cvt_float2_to_fp4x2(float2(f.x, f.y), __HIP_E2M1, hipRoundNearest)) {}
+#endif  // #if !defined(__HIP_NO_FP4_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+  __FP4_HOST_DEVICE__ operator float4() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    auto fp32x2_1 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x & 0xFFu, 1.0f /* scale */, 0);
+    auto fp32x2_2 = __builtin_amdgcn_cvt_scalef32_pk_f32_fp4(__x >> 8, 1.0f /* scale */, 0);
+#else
+    using namespace fcbx;
+    auto fp32x2_1 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>(__x & 0xFu, 0),
+                                to_float<float, Encoding::E2M1, true>((__x >> 4) & 0xFu, 0)};
+    auto fp32x2_2 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M1, true>((__x >> 8) & 0xFu, 0),
+                                to_float<float, Encoding::E2M1, true>(__x >> 12, 0)};
+#endif
+    return float4{fp32x2_1[0], fp32x2_1[1], fp32x2_2[0], fp32x2_2[1]};
+  }
+
+  __FP4_HOST_DEVICE__ operator double4() const {
+    auto fp32 = float4(*this);
+    return double4{fp32.x, fp32.y, fp32.z, fp32.w};
+  }
+#endif  // !defined(__HIP_NO_FP4_CONVERSION_OPERATORS__)
+};

projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp6.h

@@ -0,0 +1,733 @@
+/*
+Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#pragma once
+
+#include "amd_hip_mx_common.h"
+
+#include "amd_hip_fp16.h"
+#include "amd_hip_bf16.h"
+#include "amd_hip_fp8.h"
+
+#include "amd_hip_ocp_types.h"
+#include "amd_hip_ocp_host.hpp"
+#include "hip/amd_detail/amd_hip_mx_common.h"
+
+#if defined(__HIPCC_RTC__)
+#define __FP6_HOST_DEVICE__ __device__
+#define __FP6_HOST_DEVICE_STATIC__ __FP6_HOST_DEVICE__ static
+#else
+#define __FP6_HOST_DEVICE__ __host__ __device__
+#define __FP6_HOST_DEVICE_STATIC__ __FP6_HOST_DEVICE__ static inline
+#endif  // __HIPCC_RTC__
+
+typedef __hip_fp8_storage_t __hip_fp6_storage_t;
+typedef __hip_fp8x2_storage_t __hip_fp6x2_storage_t;
+typedef __hip_fp8x4_storage_t __hip_fp6x4_storage_t;
+
+static_assert(sizeof(__hip_fp6_storage_t[4]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp6x2_storage_t[2]) == sizeof(uint32_t));
+static_assert(sizeof(__hip_fp6x4_storage_t[2]) == sizeof(uint64_t));
+
+enum __hip_fp6_interpretation_t {
+  __HIP_E3M2 = 0, /**< FP6 E3M2 Type*/
+  __HIP_E2M3 = 1, /**< FP6 E2M3 Type */
+};
+
+namespace internal {
+__FP6_HOST_DEVICE_STATIC__ __amd_fp16_storage_t half_to_f16(const __half val) {
+  __half_raw tmp = val;
+  return tmp.data;
+}
+__FP6_HOST_DEVICE_STATIC__ __amd_fp16x2_storage_t half2_to_f16x2(const __half2 val) {
+  __half2_raw tmp = val;
+  return tmp.data;
+}
+__FP6_HOST_DEVICE_STATIC__ __amd_bf16_storage_t hipbf16_to_bf16(const __hip_bfloat16 val) {
+  static_assert(sizeof(__hip_bfloat16) == sizeof(__amd_bf16_storage_t));
+  union {
+    __hip_bfloat16 hip_bf16;
+    __amd_bf16_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+__FP6_HOST_DEVICE_STATIC__ __amd_bf16x2_storage_t hipbf162_to_bf16x2(const __hip_bfloat162 val) {
+  static_assert(sizeof(__hip_bfloat162) == sizeof(__amd_bf16x2_storage_t));
+  union {
+    __hip_bfloat162 hip_bf16;
+    __amd_bf16x2_storage_t bf16;
+  } u{val};
+  return u.bf16;
+}
+}  // namespace internal
+
+
+// Note: Ignore rounding input on AMD GPUs for now. At the moment AMD GPUs do not support rounding
+// modes, all the inputs are rounded to nearest or use an input to do stochastic rounding.
+// We hide the rounding variable to not trigger the unused variable compiler warning.
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t __hip_cvt_bfloat16raw_to_fp6(
+    const __hip_bfloat16_raw x, const __hip_fp6_interpretation_t fp6_interpretation,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_bf16x32_storage_t in;
+  __amd_fp6x32_storage_t out;
+  in[0] = internal::hipbf16_to_bf16(x);
+  if (fp6_interpretation == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_bf16(in, 1.0f /* scale */);
+  else if (fp6_interpretation == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_bf16(in, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation == __HIP_E2M3)
+    u.ui32 = fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M3, true>(
+        internal::hipbf16_to_bf16(x), 0);
+  else if (fp6_interpretation == __HIP_E3M2)
+    u.ui32 = fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E3M2, true>(
+        internal::hipbf16_to_bf16(x), 0);
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t __hip_cvt_bfloat16raw2_to_fp6x2(
+    const __hip_bfloat162_raw x, const __hip_fp6_interpretation_t fp6_interpretation,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_bf16x32_storage_t in;
+  in[0] = internal::hipbf16_to_bf16(x.x);
+  in[1] = internal::hipbf16_to_bf16(x.y);
+  __amd_fp6x32_storage_t out;
+  if (fp6_interpretation == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_bf16(in, 1.0f /* scale */);
+  else if (fp6_interpretation == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_bf16(in, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6x2[0];
+#else
+  if (fp6_interpretation == __HIP_E2M3) {
+    auto bf16x2 = internal::hipbf162_to_bf16x2(x);
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M3, true>(bf16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E2M3, true>(bf16x2[0], 0);
+  } else if (fp6_interpretation == __HIP_E3M2) {
+    auto bf16x2 = internal::hipbf162_to_bf16x2(x);
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E3M2, true>(bf16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_bf16_storage_t, fcbx::Encoding::E3M2, true>(bf16x2[0], 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t
+__hip_cvt_double_to_fp6(const double x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                        const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = float(x);
+  in2[0] = 0.0f;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M3, true>(float(x), 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E3M2, true>(float(x), 0);
+  }
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t
+__hip_cvt_double2_to_fp6x2(const double2 x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                           const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = float(x.x);
+  in2[0] = float(x.y);
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0] & 0x3Fu;
+  u.ui32 |= ((out[0] & 0xFC0u) << 2);
+  return u.fp6x2[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(float(x.y), 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(float(x.x), 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(float(x.y), 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(float(x.x), 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t
+__hip_cvt_float_to_fp6(const float x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                       const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = x;
+  in2[0] = 0.0f;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3)
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E2M3, true>(x, 0);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    u.ui32 = fcbx::from_float<float, fcbx::Encoding::E3M2, true>(x, 0);
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t
+__hip_cvt_float2_to_fp6x2(const float2 x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                          const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_floatx16_storage_t in1;
+  __amd_floatx16_storage_t in2;
+  __amd_fp6x32_storage_t out;
+  in1[0] = x.x;
+  in2[0] = x.y;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_fp6_f32(in1, in2, 1.0f /* scale */);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_2xpk16_bf6_f32(in1, in2, 1.0f /* scale */);
+  u.ui32 = out[0] & 0x3Fu;
+  u.ui32 |= ((out[0] & 0xFC0u) << 2);
+  return u.fp6x2[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(x.y, 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E2M3, true>(x.x, 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(x.y, 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<float, fcbx::Encoding::E3M2, true>(x.x, 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __half_raw __hip_cvt_fp6_to_halfraw(
+    const __hip_fp6_storage_t x, const __hip_fp6_interpretation_t fp6_interpretation_t) {
+  __half_raw ret;
+#if __gfx950__
+  __amd_fp16x32_storage_t out;
+  __amd_fp6x32_storage_t in;
+  in[0] = (uint32_t)x;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_fp6(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_bf6(in, 1.0f);
+  ret.data = out[0];
+#else
+  using namespace fcbx;
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    ret.data = __amd_fp16_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M3, true>(x, 0)};
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    ret.data = __amd_fp16_storage_t{to_float<__amd_fp16_storage_t, Encoding::E3M2, true>(x, 0)};
+  }
+#endif
+  return ret;
+}
+__FP6_HOST_DEVICE_STATIC__ __half2_raw __hip_cvt_fp6x2_to_halfraw2(
+    const __hip_fp6x2_storage_t x, const __hip_fp6_interpretation_t fp6_interpretation_t) {
+  __half2_raw ret;
+#if __gfx950__
+  __amd_fp16x32_storage_t out;
+  __amd_fp6x32_storage_t in;
+  in[0] = x & 0x3Fu;            // first 6 bits
+  in[0] |= (x & 0x3F00u) >> 2;  // next 6 bits
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_fp6(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f16_bf6(in, 1.0f);
+  ret.data = {out[0], out[1]};
+#else
+  using namespace fcbx;
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    ret.data =
+        __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E2M3, true>(x & 0xFFu, 0),
+                               to_float<__amd_fp16_storage_t, Encoding::E2M3, true>(x >> 8, 0)};
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    ret.data =
+        __amd_fp16x2_storage_t{to_float<__amd_fp16_storage_t, Encoding::E3M2, true>(x & 0xFFu, 0),
+                               to_float<__amd_fp16_storage_t, Encoding::E3M2, true>(x >> 8, 0)};
+  }
+#endif
+  return ret;
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6_storage_t
+__hip_cvt_halfraw_to_fp6(const __half_raw x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+                         const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6_storage_t fp6[4];
+  } u{0};
+#if __gfx950__
+  __amd_fp16x32_storage_t in;
+  __amd_fp6x32_storage_t out;
+  in[0] = x.data;
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_f16(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_f16(in, 1.0f);
+  u.ui32 = out[0];
+  return u.fp6[0];
+#else
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 = fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M3, true>(
+        internal::half_to_f16(x), 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 = fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E3M2, true>(
+        internal::half_to_f16(x), 0);
+  }
+  return u.fp6[0];
+#endif
+}
+__FP6_HOST_DEVICE_STATIC__ __hip_fp6x2_storage_t __hip_cvt_halfraw2_to_fp6x2(
+    const __half2_raw x, const __hip_fp6_interpretation_t fp6_interpretation_t,
+    const enum hipRoundMode /* rounding */) {
+  union {
+    uint32_t ui32;
+    __hip_fp6x2_storage_t fp6x2[2];
+  } u{0};
+#if __gfx950__
+  __amd_fp16x32_storage_t in;
+  __amd_fp6x32_storage_t out;
+  in[0] = x.data[0];
+  in[1] = x.data[1];
+  if (fp6_interpretation_t == __HIP_E2M3)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_fp6_f16(in, 1.0f);
+  else if (fp6_interpretation_t == __HIP_E3M2)
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf6_f16(in, 1.0f);
+  u.ui32 = out[0] & 0x3Fu;
+  u.ui32 |= ((out[0] & 0xFC0u) << 2);
+  return u.fp6x2[0];
+#else
+  auto fp16x2 = internal::half2_to_f16x2(x);
+  if (fp6_interpretation_t == __HIP_E2M3) {
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M3, true>(fp16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E2M3, true>(fp16x2[0], 0);
+  } else if (fp6_interpretation_t == __HIP_E3M2) {
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E3M2, true>(fp16x2[1], 0);
+    u.ui32 <<= 8;
+    u.ui32 |= fcbx::from_float<__amd_fp16_storage_t, fcbx::Encoding::E3M2, true>(fp16x2[0], 0);
+  }
+  return u.fp6x2[0];
+#endif
+}
+
+//======================== structs ====================
+struct __hip_fp6_e2m3 {
+  __hip_fp6_storage_t __x;
+
+  __FP6_HOST_DEVICE__ __hip_fp6_e2m3() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const __half f)
+      : __x(__hip_cvt_halfraw_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const __hip_bfloat16 f)
+      : __x(__hip_cvt_bfloat16raw_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const double f)
+      : __x(__hip_cvt_double_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const float f)
+      : __x(__hip_cvt_float_to_fp6(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e2m3(const unsigned short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E2M3, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half_raw() const {
+    return __hip_cvt_fp6_to_halfraw(__x, __HIP_E2M3);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat16_raw() const {
+    static_assert(sizeof(__hip_bfloat16_raw) == sizeof(__amd_bf16_storage_t));
+    union {
+      __hip_bfloat16_raw bf16_raw;
+      __amd_bf16_storage_t bf16;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_fp6(in, 1.0f /* scale */);
+    u.bf16 = out[0];
+#else
+    using namespace fcbx;
+    u.bf16 = to_float<__amd_bf16_storage_t, Encoding::E2M3, true>(__x, 0);
+#endif
+    return u.bf16_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6(in, 1.0f /* scale */);
+    auto ret = out[0];
+#else
+    using namespace fcbx;
+    float ret{to_float<float, Encoding::E2M3, true>(__x, 0)};
+#endif
+    return ret;
+  }
+  __FP6_HOST_DEVICE__ operator double() const { return double(float(*this)); }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6_e3m2 {
+  __hip_fp6_storage_t __x;
+
+  __FP6_HOST_DEVICE__ __hip_fp6_e3m2() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const __half f)
+      : __x(__hip_cvt_halfraw_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const __hip_bfloat16 f)
+      : __x(__hip_cvt_bfloat16raw_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const double f)
+      : __x(__hip_cvt_double_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const float f)
+      : __x(__hip_cvt_float_to_fp6(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned long long int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6_e3m2(const unsigned short int val)
+      : __x(__hip_cvt_float_to_fp6(float(val), __HIP_E3M2, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half_raw() const {
+    return __hip_cvt_fp6_to_halfraw(__x, __HIP_E3M2);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat16_raw() const {
+    static_assert(sizeof(__hip_bfloat16_raw) == sizeof(__amd_bf16_storage_t));
+    union {
+      __hip_bfloat16_raw bf16_raw;
+      __amd_bf16_storage_t bf16;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_bf6(in, 1.0f /* scale */);
+    u.bf16 = out[0];
+#else
+    using namespace fcbx;
+    u.bf16 = to_float<__amd_bf16_storage_t, Encoding::E3M2, true>(__x, 0);
+#endif
+    return u.bf16_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = (uint32_t)__x;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6(in, 1.0f /* scale */);
+    auto ret = out[0];
+#else
+    using namespace fcbx;
+    float ret{to_float<float, Encoding::E3M2, true>(__x, 0)};
+#endif
+    return ret;
+  }
+  __FP6_HOST_DEVICE__ operator double() const { return double(float(*this)); }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x2_e2m3 {
+  __hip_fp6x2_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x2_e2m3() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const __half2 f)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const __hip_bfloat162 f)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const double2 f)
+      : __x(__hip_cvt_double2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e2m3(const float2 f)
+      : __x(__hip_cvt_float2_to_fp6x2(f, __HIP_E2M3, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half2_raw() const {
+    return __hip_cvt_fp6x2_to_halfraw2(__x, __HIP_E2M3);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat162_raw() const {
+    static_assert(sizeof(__hip_bfloat162_raw) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat162_raw bf162_raw;
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_fp6(in, 1.0f /* scale */);
+    u.bf16x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E2M3, true>(__x & 0xFFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E2M3, true>(__x >> 8, 0)};
+#endif
+    return u.bf162_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6(in, 1.0f /* scale */);
+    auto fp32x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E2M3, true>(__x & 0xFFu, 0),
+                                          to_float<float, Encoding::E2M3, true>(__x >> 8, 0)};
+#endif
+    return float2(fp32x2[0], fp32x2[1]);
+  }
+  __FP6_HOST_DEVICE__ operator double2() const {
+    auto fp32 = float2(*this);
+    return double2(fp32.x, fp32.y);
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x2_e3m2 {
+  __hip_fp6x2_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x2_e3m2() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const __half2 f)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const __hip_bfloat162 f)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const double2 f)
+      : __x(__hip_cvt_double2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x2_e3m2(const float2 f)
+      : __x(__hip_cvt_float2_to_fp6x2(f, __HIP_E3M2, hipRoundNearest)) {}
+#endif  //! defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator __half2_raw() const {
+    return __hip_cvt_fp6x2_to_halfraw2(__x, __HIP_E3M2);
+  }
+  __FP6_HOST_DEVICE__ operator __hip_bfloat162_raw() const {
+    static_assert(sizeof(__hip_bfloat162_raw) == sizeof(__amd_bf16x2_storage_t));
+    union {
+      __hip_bfloat162_raw bf162_raw;
+      __amd_bf16x2_storage_t bf16x2;
+    } u;
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_bf16x32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_bf16_bf6(in, 1.0f /* scale */);
+    u.bf16x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    u.bf16x2 =
+        __amd_bf16x2_storage_t{to_float<__amd_bf16_storage_t, Encoding::E3M2, true>(__x & 0xFFu, 0),
+                               to_float<__amd_bf16_storage_t, Encoding::E3M2, true>(__x >> 8, 0)};
+#endif
+    return u.bf162_raw;
+  }
+  __FP6_HOST_DEVICE__ operator float2() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;          // first 6 bits
+    in[0] |= (__x & FC00u) >> 2;  // next 6 bits
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6(in, 1.0f /* scale */);
+    auto fp32x2 = {out[0], out[1]};
+#else
+    using namespace fcbx;
+    auto fp32x2 = __amd_floatx2_storage_t{to_float<float, Encoding::E3M2, true>(__x & 0xFFu, 0),
+                                          to_float<float, Encoding::E3M2, true>(__x >> 8, 0)};
+#endif
+    return float2(fp32x2[0], fp32x2[1]);
+  }
+  __FP6_HOST_DEVICE__ operator double2() const {
+    auto fp32 = float2(*this);
+    return double2(fp32.x, fp32.y);
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x4_e2m3 {
+  __hip_fp6x4_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x4_e2m3() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const __half2 low, const __half2 high)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(high, __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_halfraw2_to_fp6x2(low, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const __hip_bfloat162 low,
+                                                       const __hip_bfloat162 high)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(high, __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_bfloat16raw2_to_fp6x2(low, __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const double4 f)
+      : __x(__hip_cvt_double2_to_fp6x2(double2(f.z, f.w), __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_double2_to_fp6x2(double2(f.x, f.y), __HIP_E2M3, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e2m3(const float4 f)
+      : __x(__hip_cvt_float2_to_fp6x2(float2(f.z, f.w), __HIP_E2M3, hipRoundNearest) << 16 |
+            __hip_cvt_float2_to_fp6x2(float2(f.x, f.y), __HIP_E2M3, hipRoundNearest)) {}
+#endif  // !defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator float4() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;                 // first 6 bits
+    in[0] |= ((__x >> 8) & 0x3Fu) << 6;  // second 6 bits
+    in[0] |= ((__x >> 16) & 0x3Fu) << 12;
+    in[0] |= ((__x >> 24) & 0x3Fu) << 18;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_fp6(in, 1.0f /* scale */);
+    auto fp32x2_1 = {out[0], out[1]};
+    auto fp32x2_2 = {out[2], out[3]};
+#else
+    using namespace fcbx;
+    auto fp32x2_1 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M3, true>(__x & 0xFFu, 0),
+                                to_float<float, Encoding::E2M3, true>((__x >> 8) & 0xFFu, 0)};
+    auto fp32x2_2 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E2M3, true>((__x >> 16) & 0xFFu, 0),
+                                to_float<float, Encoding::E2M3, true>(__x >> 24, 0)};
+#endif
+    return float4{fp32x2_1[0], fp32x2_1[1], fp32x2_2[0], fp32x2_2[1]};
+  }
+  __FP6_HOST_DEVICE__ operator double4() const {
+    auto fp32 = float4(*this);
+    return double4{fp32.x, fp32.y, fp32.z, fp32.w};
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};
+
+struct __hip_fp6x4_e3m2 {
+  __hip_fp6x4_storage_t __x;
+  __FP6_HOST_DEVICE__ inline __hip_fp6x4_e3m2() = default;
+#if !defined(__HIP_NO_FP6_CONVERSIONS__)
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const __half2 high, const __half2 low)
+      : __x(__hip_cvt_halfraw2_to_fp6x2(high, __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_halfraw2_to_fp6x2(low, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const __hip_bfloat162 high,
+                                                       const __hip_bfloat162 low)
+      : __x(__hip_cvt_bfloat16raw2_to_fp6x2(high, __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_bfloat16raw2_to_fp6x2(low, __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const double4 f)
+      : __x(__hip_cvt_double2_to_fp6x2(double2(f.z, f.w), __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_double2_to_fp6x2(double2(f.x, f.y), __HIP_E3M2, hipRoundNearest)) {}
+  __FP6_HOST_DEVICE__ inline explicit __hip_fp6x4_e3m2(const float4 f)
+      : __x(__hip_cvt_float2_to_fp6x2(float2(f.z, f.w), __HIP_E3M2, hipRoundNearest) << 16 |
+            __hip_cvt_float2_to_fp6x2(float2(f.x, f.y), __HIP_E3M2, hipRoundNearest)) {}
+#endif  //! defined(__HIP_NO_FP6_CONVERSIONS__)
+#if !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+  __FP6_HOST_DEVICE__ operator float4() const {
+#if HIP_ENABLE_GFX950_OCP_BUILTINS
+    __amd_fp6x32_storage_t in;
+    __amd_floatx32_storage_t out;
+    in[0] = __x & 0x3Fu;                 // first 6 bits
+    in[0] |= ((__x >> 8) & 0x3Fu) << 6;  // second 6 bits
+    in[0] |= ((__x >> 16) & 0x3Fu) << 12;
+    in[0] |= ((__x >> 24) & 0x3Fu) << 18;
+    out = __builtin_amdgcn_cvt_scalef32_pk32_f32_bf6(in, 1.0f /* scale */);
+    auto fp32x2_1 = {out[0], out[1]};
+    auto fp32x2_2 = {out[2], out[3]};
+#else
+    using namespace fcbx;
+    auto fp32x2_1 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E3M2, true>(__x & 0xFFu, 0),
+                                to_float<float, Encoding::E3M2, true>((__x >> 8) & 0xFFu, 0)};
+    auto fp32x2_2 =
+        __amd_floatx2_storage_t{to_float<float, Encoding::E3M2, true>((__x >> 16) & 0xFFu, 0),
+                                to_float<float, Encoding::E3M2, true>(__x >> 24, 0)};
+#endif
+    return float4{fp32x2_1[0], fp32x2_1[1], fp32x2_2[0], fp32x2_2[1]};
+  }
+  __FP6_HOST_DEVICE__ operator double4() const {
+    auto fp32 = float4(*this);
+    return double4{fp32.x, fp32.y, fp32.z, fp32.w};
+  }
+#endif  // !defined(__HIP_NO_FP6_CONVERSION_OPERATORS__)
+};

projects/clr/hipamd/include/hip/amd_detail/amd_hip_gl_interop.h

@@ -0,0 +1,110 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_AMD_HIP_GL_INTEROP_H
+#define HIP_INCLUDE_AMD_HIP_GL_INTEROP_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/**
+ *
+ * @addtogroup GlobalDefs
+ * @{
+ *
+ */
+
+/**
+ * HIP Devices used by current OpenGL Context.
+ */
+typedef enum hipGLDeviceList {
+    hipGLDeviceListAll = 1,           ///< All hip devices used by current OpenGL context.
+    hipGLDeviceListCurrentFrame = 2,  ///< Hip devices used by current OpenGL context in current
+                                    ///< frame
+    hipGLDeviceListNextFrame = 3      ///< Hip devices used by current OpenGL context in next
+                                    ///< frame.
+} hipGLDeviceList;
+
+
+/** GLuint as uint.*/
+typedef unsigned int GLuint;
+/** GLenum as uint.*/
+typedef unsigned int GLenum;
+/**
+* @}
+*/
+
+/**
+ * @defgroup GL OpenGL Interoperability
+ * @ingroup API
+ * @{
+ * This section describes OpenGL interoperability functions of HIP runtime API.
+ */
+
+/**
+ * @brief Queries devices associated with the current OpenGL context.
+ *
+ * @param [out] pHipDeviceCount - Pointer of number of devices on the current GL context.
+ * @param [out] pHipDevices - Pointer of devices on the current OpenGL context.
+ * @param [in] hipDeviceCount - Size of device.
+ * @param [in] deviceList - The setting of devices. It could be either hipGLDeviceListCurrentFrame
+ * for the devices used to render the current frame, or hipGLDeviceListAll for all devices.
+ * The default setting is Invalid deviceList value.
+ *
+ * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorNotSupported
+ *
+ */
+hipError_t hipGLGetDevices(unsigned int* pHipDeviceCount, int* pHipDevices,
+                           unsigned int hipDeviceCount, hipGLDeviceList deviceList);
+/**
+ * @brief Registers a GL Buffer for interop and returns corresponding graphics resource.
+ *
+ * @param [out] resource - Returns pointer of graphics resource.
+ * @param [in] buffer - Buffer to be registered.
+ * @param [in] flags - Register flags.
+ * 
+ * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorUnknown, #hipErrorInvalidResourceHandle
+ *
+ */
+hipError_t hipGraphicsGLRegisterBuffer(hipGraphicsResource** resource, GLuint buffer,
+                                       unsigned int flags);
+/**
+ * @brief Register a GL Image for interop and returns the corresponding graphic resource.
+ *
+ * @param [out] resource - Returns pointer of graphics resource.
+ * @param [in] image - Image to be registered.
+ * @param [in] target - Valid target value Id.
+ * @param [in] flags - Register flags.
+ *
+ * @returns #hipSuccess, #hipErrorInvalidValue, #hipErrorUnknown, #hipErrorInvalidResourceHandle
+ *
+ */
+hipError_t hipGraphicsGLRegisterImage(hipGraphicsResource** resource, GLuint image,
+                                      GLenum target, unsigned int flags);
+/**
+* @}
+*/
+#if defined(__cplusplus)
+}
+#endif /* __cplusplus */
+#endif /* HIP_INCLUDE_AMD_HIP_GL_INTEROP_H */

projects/clr/hipamd/include/hip/amd_detail/amd_hip_math_constants.h

@@ -0,0 +1,124 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef AMD_HIP_MATH_CONSTANTS_H
+#define AMD_HIP_MATH_CONSTANTS_H
+
+// single precision constants
+#define HIP_INF_F            __int_as_float(0x7f800000U)
+#define HIP_NAN_F            __int_as_float(0x7fffffffU)
+#define HIP_MIN_DENORM_F     __int_as_float(0x00000001U)
+#define HIP_MAX_NORMAL_F     __int_as_float(0x7f7fffffU)
+#define HIP_NEG_ZERO_F       __int_as_float(0x80000000U)
+#define HIP_ZERO_F           0.0F
+#define HIP_ONE_F            1.0F
+#define HIP_SQRT_HALF_F      0.707106781F
+#define HIP_SQRT_HALF_HI_F   0.707106781F
+#define HIP_SQRT_HALF_LO_F   1.210161749e-08F
+#define HIP_SQRT_TWO_F       1.414213562F
+#define HIP_THIRD_F          0.333333333F
+#define HIP_PIO4_F           0.785398163F
+#define HIP_PIO2_F           1.570796327F
+#define HIP_3PIO4_F          2.356194490F
+#define HIP_2_OVER_PI_F      0.636619772F
+#define HIP_SQRT_2_OVER_PI_F 0.797884561F
+#define HIP_PI_F             3.141592654F
+#define HIP_L2E_F            1.442695041F
+#define HIP_L2T_F            3.321928094F
+#define HIP_LG2_F            0.301029996F
+#define HIP_LGE_F            0.434294482F
+#define HIP_LN2_F            0.693147181F
+#define HIP_LNT_F            2.302585093F
+#define HIP_LNPI_F           1.144729886F
+#define HIP_TWO_TO_M126_F    1.175494351e-38F
+#define HIP_TWO_TO_126_F     8.507059173e37F
+#define HIP_NORM_HUGE_F      3.402823466e38F
+#define HIP_TWO_TO_23_F      8388608.0F
+#define HIP_TWO_TO_24_F      16777216.0F
+#define HIP_TWO_TO_31_F      2147483648.0F
+#define HIP_TWO_TO_32_F      4294967296.0F
+#define HIP_REMQUO_BITS_F    3U
+#define HIP_REMQUO_MASK_F    (~((~0U)<<HIP_REMQUO_BITS_F))
+#define HIP_TRIG_PLOSS_F     105615.0F
+
+// double precision constants
+#define HIP_INF              __longlong_as_double(0x7ff0000000000000ULL)
+#define HIP_NAN              __longlong_as_double(0xfff8000000000000ULL)
+#define HIP_NEG_ZERO         __longlong_as_double(0x8000000000000000ULL)
+#define HIP_MIN_DENORM       __longlong_as_double(0x0000000000000001ULL)
+#define HIP_ZERO             0.0
+#define HIP_ONE              1.0
+#define HIP_SQRT_TWO         1.4142135623730951e+0
+#define HIP_SQRT_HALF        7.0710678118654757e-1
+#define HIP_SQRT_HALF_HI     7.0710678118654757e-1
+#define HIP_SQRT_HALF_LO   (-4.8336466567264567e-17)
+#define HIP_THIRD            3.3333333333333333e-1
+#define HIP_TWOTHIRD         6.6666666666666667e-1
+#define HIP_PIO4             7.8539816339744828e-1
+#define HIP_PIO4_HI          7.8539816339744828e-1
+#define HIP_PIO4_LO          3.0616169978683830e-17
+#define HIP_PIO2             1.5707963267948966e+0
+#define HIP_PIO2_HI          1.5707963267948966e+0
+#define HIP_PIO2_LO          6.1232339957367660e-17
+#define HIP_3PIO4            2.3561944901923448e+0
+#define HIP_2_OVER_PI        6.3661977236758138e-1
+#define HIP_PI               3.1415926535897931e+0
+#define HIP_PI_HI            3.1415926535897931e+0
+#define HIP_PI_LO            1.2246467991473532e-16
+#define HIP_SQRT_2PI         2.5066282746310007e+0
+#define HIP_SQRT_2PI_HI      2.5066282746310007e+0
+#define HIP_SQRT_2PI_LO    (-1.8328579980459167e-16)
+#define HIP_SQRT_PIO2        1.2533141373155003e+0
+#define HIP_SQRT_PIO2_HI     1.2533141373155003e+0
+#define HIP_SQRT_PIO2_LO   (-9.1642899902295834e-17)
+#define HIP_SQRT_2OPI        7.9788456080286536e-1
+#define HIP_L2E              1.4426950408889634e+0
+#define HIP_L2E_HI           1.4426950408889634e+0
+#define HIP_L2E_LO           2.0355273740931033e-17
+#define HIP_L2T              3.3219280948873622e+0
+#define HIP_LG2              3.0102999566398120e-1
+#define HIP_LG2_HI           3.0102999566398120e-1
+#define HIP_LG2_LO         (-2.8037281277851704e-18)
+#define HIP_LGE              4.3429448190325182e-1
+#define HIP_LGE_HI           4.3429448190325182e-1
+#define HIP_LGE_LO           1.09831965021676510e-17
+#define HIP_LN2              6.9314718055994529e-1
+#define HIP_LN2_HI           6.9314718055994529e-1
+#define HIP_LN2_LO           2.3190468138462996e-17
+#define HIP_LNT              2.3025850929940459e+0
+#define HIP_LNT_HI           2.3025850929940459e+0
+#define HIP_LNT_LO         (-2.1707562233822494e-16)
+#define HIP_LNPI             1.1447298858494002e+0
+#define HIP_LN2_X_1024       7.0978271289338397e+2
+#define HIP_LN2_X_1025       7.1047586007394398e+2
+#define HIP_LN2_X_1075       7.4513321910194122e+2
+#define HIP_LG2_X_1024       3.0825471555991675e+2
+#define HIP_LG2_X_1075       3.2360724533877976e+2
+#define HIP_TWO_TO_23        8388608.0
+#define HIP_TWO_TO_52        4503599627370496.0
+#define HIP_TWO_TO_53        9007199254740992.0
+#define HIP_TWO_TO_54        18014398509481984.0
+#define HIP_TWO_TO_M54       5.5511151231257827e-17
+#define HIP_TWO_TO_M1022     2.22507385850720140e-308
+#define HIP_TRIG_PLOSS       2147483648.0
+#define HIP_DBL2INT_CVT      6755399441055744.0
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_hip_mx_common.h

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#pragma once
+
+enum hipRoundMode {
+  hipRoundNearest = 0,
+  hipRoundZero = 1,
+  hipRoundPosInf = 2,
+  hipRoundMinInf = 3,
+};

projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_host.hpp

@@ -0,0 +1,985 @@
+/*
+Copyright © Advanced Micro Devices, Inc., or its affiliates.
+ 
+SPDX-License-Identifier: MIT
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "amd_hip_ocp_types.h"
+
+#if !defined(__HIPCC_RTC__)
+#include <cstdint>
+#include <limits>
+#include <cstdlib>
+#include <cmath>
+#endif
+
+namespace fcbx {
+constexpr int8_t OCP_SCALE_EXP_NAN = -128;
+
+enum class Encoding : size_t {
+  E2M1 = 0,
+  E2M3,
+  E3M2,
+  E4M3,
+  E4M3Mx,
+  E4M3Nanoo,
+  E5M2,
+  E5M2Mx,
+  E5M2Nanoo,
+
+  E5M10,  // FP16
+  E8M7,   // BF16
+
+  IEEE754,
+
+  // Keep this one last
+  NumEncodings,
+};
+enum fp16 : uint16_t {};
+enum bf16 : uint16_t {};
+
+struct Float {
+  int32_t ExpBias;
+  uint32_t ExpBits;
+  uint32_t ExpMask;
+  uint32_t ManBits;
+  uint32_t ManMask;
+  int32_t MaxExp;
+  int32_t MinExp;
+  bool MxScale;
+  bool HasNaN;
+  bool HasInf;
+};
+
+static const float ieee754_nan = std::numeric_limits<float>::quiet_NaN();
+static const float ieee754_inf = std::numeric_limits<float>::infinity();
+
+__OCP_FP_HOST_DEVICE_STATIC__ uint32_t U32(float f) {
+  static_assert(sizeof(uint32_t) == sizeof(float));
+  union {
+    float f32;
+    uint32_t ui32;
+  } u{f};
+  return u.ui32;
+}
+
+__OCP_FP_HOST_DEVICE_STATIC__ float F32(uint32_t u32) {
+  static_assert(sizeof(uint32_t) == sizeof(float));
+  union {
+    uint32_t ui32;
+    float f32;
+  } u{u32};
+  return u.f32;
+}
+
+constexpr __OCP_FP_HOST_DEVICE_STATIC__ uint32_t bitmask(uint32_t bits) {
+  if (bits < 1) return 0;
+  return ((uint32_t)1 << bits) - 1;
+}
+
+constexpr std::array<Float, (size_t)Encoding::NumEncodings> init() {
+  std::array<Float, (size_t)Encoding::NumEncodings> a{};
+
+  a[(size_t)Encoding::E2M1] = {
+      .ExpBias = 1,
+      .ExpBits = 2,
+      .ExpMask = bitmask(2),
+      .ManBits = 1,
+      .ManMask = bitmask(1),
+      .MaxExp = 2,
+      .MinExp = 0,
+      .MxScale = true,
+      .HasNaN = false,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E2M3] = {
+      .ExpBias = 1,
+      .ExpBits = 2,
+      .ExpMask = bitmask(2),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 2,
+      .MinExp = 0,
+      .MxScale = true,
+      .HasNaN = false,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E3M2] = {
+      .ExpBias = 3,
+      .ExpBits = 3,
+      .ExpMask = bitmask(3),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 4,
+      .MinExp = -2,
+      .MxScale = true,
+      .HasNaN = false,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E4M3] = {
+      .ExpBias = 7,
+      .ExpBits = 4,
+      .ExpMask = bitmask(4),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 8,
+      .MinExp = -6,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E4M3Mx] = {
+      .ExpBias = 7,
+      .ExpBits = 4,
+      .ExpMask = bitmask(4),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 8,
+      .MinExp = -6,
+      .MxScale = true,
+      .HasNaN = true,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E4M3Nanoo] = {
+      .ExpBias = 8,
+      .ExpBits = 4,
+      .ExpMask = bitmask(4),
+      .ManBits = 3,
+      .ManMask = bitmask(3),
+      .MaxExp = 7,
+      .MinExp = -7,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = false,
+  };
+
+  a[(size_t)Encoding::E5M2] = {
+      .ExpBias = 15,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 15,
+      .MinExp = -14,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E5M2Mx] = {
+      .ExpBias = 15,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 15,
+      .MinExp = -14,
+      .MxScale = true,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E5M2Nanoo] = {
+      .ExpBias = 16,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 2,
+      .ManMask = bitmask(2),
+      .MaxExp = 15,
+      .MinExp = -15,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E5M10] = {
+      .ExpBias = 15,
+      .ExpBits = 5,
+      .ExpMask = bitmask(5),
+      .ManBits = 10,
+      .ManMask = bitmask(10),
+      .MaxExp = 15,
+      .MinExp = -14,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::E8M7] = {
+      .ExpBias = 127,
+      .ExpBits = 8,
+      .ExpMask = bitmask(8),
+      .ManBits = 7,
+      .ManMask = bitmask(7),
+      .MaxExp = 127,
+      .MinExp = -126,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  a[(size_t)Encoding::IEEE754] = {
+      .ExpBias = 127,
+      .ExpBits = 8,
+      .ExpMask = bitmask(8),
+      .ManBits = 23,
+      .ManMask = bitmask(23),
+      .MaxExp = 127,
+      .MinExp = -126,
+      .MxScale = false,
+      .HasNaN = true,
+      .HasInf = true,
+  };
+
+  return a;
+}
+
+static constexpr auto encodings = init();
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t exponentbits(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  return (val >> enc.ManBits) & enc.ExpMask;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t mantissa(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  return val & enc.ManMask;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool issubnorm(uint32_t val) {
+  switch (E) {
+    default:
+      return exponentbits<E, sat>(val) == 0 && mantissa<E, sat>(val) != 0;
+  }
+
+  __builtin_trap();
+  // Unreachable
+  return false;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ int32_t exponent(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  auto unbiased_exp = exponentbits<E, sat>(val);
+  unbiased_exp = issubnorm<E, sat>(val) ? 1 : unbiased_exp;
+  return (int32_t)unbiased_exp - enc.ExpBias;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t signbit(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  return (val >> (enc.ExpBits + enc.ManBits)) & 1;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t nan(uint32_t sign) {
+  const auto& enc = encodings[(size_t)E];
+
+  switch (E) {
+    case Encoding::E2M1:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0b0111;
+    case Encoding::E2M3:
+    case Encoding::E3M2:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0b011111;
+    case Encoding::E4M3:
+    case Encoding::E4M3Mx:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0x7f;
+    case Encoding::E5M2:
+    case Encoding::E5M2Mx:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0x7e;
+    case Encoding::E4M3Nanoo:
+    case Encoding::E5M2Nanoo:
+      return 0b10000000;
+    case Encoding::E5M10:
+    case Encoding::E8M7:
+      return (sign << (enc.ExpBits + enc.ManBits)) | (enc.ExpMask << enc.ManBits) | enc.ManMask;
+    case Encoding::IEEE754:
+      return U32(sign ? std::copysign(ieee754_nan, -1.0F) : ieee754_nan);
+    default:
+      __builtin_trap();
+      return 0;
+  }
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t zero(uint32_t sign) {
+  const auto& enc = encodings[(size_t)E];
+
+  switch (E) {
+    case Encoding::E2M1:
+    case Encoding::E2M3:
+    case Encoding::E3M2:
+    case Encoding::E4M3:
+    case Encoding::E4M3Mx:
+    case Encoding::E5M2:
+    case Encoding::E5M2Mx:
+    case Encoding::E5M10:
+    case Encoding::E8M7:
+      return (sign << (enc.ExpBits + enc.ManBits)) | 0;
+    case Encoding::E4M3Nanoo:
+    case Encoding::E5M2Nanoo:
+      return 0;
+    case Encoding::IEEE754:
+      return U32(sign ? std::copysign(0.0F, -1.0F) : 0.0F);
+    default:
+      __builtin_trap();
+      return 0;
+  }
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ uint32_t inf(uint32_t sign) {
+  const auto& enc = encodings[(size_t)E];
+
+  switch (E) {
+    case Encoding::E2M1:
+    case Encoding::E2M3:
+    case Encoding::E3M2:
+      return nan<E, sat>(sign);
+    case Encoding::E4M3:
+    case Encoding::E4M3Mx:
+    case Encoding::E4M3Nanoo:
+    case Encoding::E5M2Nanoo:
+      if constexpr (sat) {
+        sign <<= enc.ExpBits + enc.ManBits;
+        return sign | 0b01111111;
+      }
+
+      return nan<E, sat>(sign);
+    case Encoding::E5M2:
+    case Encoding::E5M2Mx:
+      sign <<= enc.ExpBits + enc.ManBits;
+      if constexpr (sat) {
+        return sign | 0b01111011;
+      }
+
+      return sign | 0b01111100;
+    case Encoding::E5M10:
+    case Encoding::E8M7:
+      sign <<= enc.ExpBits + enc.ManBits;
+      return sign | (enc.ExpMask << enc.ManBits) | 0;
+    case Encoding::IEEE754:
+      return U32(sign ? std::copysign(ieee754_inf, -1.0F) : ieee754_inf);
+    default:
+      __builtin_trap();
+      return 0;
+  }
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool isnan(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  if (!enc.HasNaN) return false;
+
+  if constexpr (E == Encoding::E4M3Mx || E == Encoding::E4M3 || E == Encoding::E4M3Nanoo ||
+                E == Encoding::E5M2Nanoo)
+    return nan<E, sat>(signbit<E, sat>(val)) == val;
+
+  return exponentbits<E, sat>(val) == enc.ExpMask && mantissa<E, sat>(val) != 0;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool isinf(uint32_t val) {
+  const auto& enc = encodings[(size_t)E];
+  if (!enc.HasInf) return false;
+
+  if constexpr (E == Encoding::E5M10 || E == Encoding::E8M7) {
+    return exponentbits<E, sat>(val) == enc.ExpMask && mantissa<E, sat>(val) == 0;
+  }
+
+  return inf<E, sat>(signbit<E, sat>(val)) == val;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool iszero(uint32_t val) {
+  return zero<E, sat>(signbit<E, sat>(val)) == val;
+}
+
+template <Encoding E, bool sat> __OCP_FP_HOST_DEVICE_STATIC__ bool inrange(uint32_t val) {
+  return !(isnan<E, sat>(val) || isinf<E, sat>(val));
+}
+
+template <typename T> __OCP_FP_HOST_DEVICE_STATIC__ T makenan(Encoding E, uint32_t sign) {
+  switch (E) {
+    case Encoding::E5M10:
+      return (T)nan<Encoding::E5M10, false>(sign);
+    case Encoding::E8M7:
+      return (T)nan<Encoding::E8M7, false>(sign);
+    case Encoding::IEEE754:
+      return (T)F32(nan<Encoding::IEEE754, false>(sign));
+    default:
+      __builtin_trap();
+      // Unreachable
+      return T();
+  }
+}
+
+template <typename T> __OCP_FP_HOST_DEVICE_STATIC__ T makeinf(Encoding E, uint32_t sign) {
+  switch (E) {
+    case Encoding::E5M10:
+      return (T)inf<Encoding::E5M10, false>(sign);
+    case Encoding::E8M7:
+      return (T)inf<Encoding::E8M7, false>(sign);
+    case Encoding::IEEE754:
+      return (T)F32(inf<Encoding::IEEE754, false>(sign));
+    default:
+      __builtin_trap();
+      // Unreachable
+      return T();
+  }
+}
+
+template <typename T> __OCP_FP_HOST_DEVICE_STATIC__ T makezero(Encoding E, uint32_t sign) {
+  switch (E) {
+    case Encoding::E5M10:
+      return (T)zero<Encoding::E5M10, false>(sign);
+    case Encoding::E8M7:
+      return (T)zero<Encoding::E8M7, false>(sign);
+    case Encoding::IEEE754:
+      return (T)F32(zero<Encoding::IEEE754, false>(sign));
+    default:
+      __builtin_trap();
+      // Unreachable
+      return T();
+  }
+}
+
+template <typename T, Encoding E, bool sat>
+__OCP_FP_HOST_DEVICE_STATIC__ T to_float(uint32_t u32, int8_t scale_exp) {
+  // We do not support bf16/fp16 <-> float
+  static_assert(E != Encoding::IEEE754 && E != Encoding::E5M10 && E != Encoding::E8M7);
+
+  const auto& enc = encodings[(size_t)E];
+  const auto dstE = []() -> Encoding {
+    if constexpr (std::is_same<T, float>())
+      return Encoding::IEEE754;
+    else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+      return Encoding::E5M10;
+    else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+      return Encoding::E8M7;
+    else
+      __builtin_trap();
+  }();
+  const auto& dstEnc = encodings[(size_t)dstE];
+
+  if (isnan<E, sat>(u32) || (enc.MxScale && scale_exp == OCP_SCALE_EXP_NAN))
+    return makenan<T>(dstE, signbit<E, sat>(u32));
+
+  if (isinf<E, sat>(u32)) return makeinf<T>(dstE, signbit<E, sat>(u32));
+
+  if (iszero<E, sat>(u32)) return makezero<T>(dstE, signbit<E, sat>(u32));
+
+  auto dstMan = mantissa<E, sat>(u32) << (dstEnc.ManBits - enc.ManBits);
+  auto dstExp = (uint32_t)(exponent<E, sat>(u32) + dstEnc.ExpBias);
+  dstExp &= dstEnc.ExpMask;
+
+  if (issubnorm<E, sat>(u32)) {
+    auto leadbit = (uint32_t)1 << dstEnc.ManBits;
+    while ((dstMan & leadbit) == 0) {
+      dstMan <<= 1;
+      dstExp -= 1;
+    }
+
+    dstMan &= dstEnc.ManMask;
+  }
+
+  auto sign = signbit<E, sat>(u32) << (dstEnc.ExpBits + dstEnc.ManBits);
+
+  if (enc.MxScale) {
+    int32_t exp = dstExp - dstEnc.ExpBias;
+    int32_t tmp = exp + (int32_t)scale_exp;
+    size_t diff = abs(tmp - dstEnc.MinExp);
+
+
+    if (tmp < dstEnc.MinExp) {
+      if (diff > dstEnc.ManBits + 1) return makezero<T>(dstE, signbit<E, sat>(u32));
+
+      dstExp = 0;  // Subnormal
+      dstMan |= (uint32_t)1 << dstEnc.ManBits;
+
+      auto roundBitShift = diff - 1;
+      auto roundBit = (dstMan & ((uint32_t)1 << roundBitShift)) != 0;
+      auto stickyMask = ((uint32_t)1 << roundBitShift) - 1;
+      auto stickyBits = dstMan & stickyMask;
+      auto odd = (dstMan & ((uint32_t)1 << diff)) != 0;
+
+      dstMan >>= diff;
+
+      if ((roundBit && stickyBits != 0) || (roundBit && odd)) {
+        ++dstMan;
+        if ((dstMan & ((uint32_t)1 << dstEnc.ManBits)) != 0) ++dstExp;
+      }
+
+      dstMan &= dstEnc.ManMask;
+    } else {
+      dstExp = (uint32_t)(exp + scale_exp + dstEnc.ExpBias);
+
+      // Overflow: return infinity (gfx950 HW behavior)
+      if (dstExp >= dstEnc.ExpMask) return makeinf<T>(dstE, signbit<E, sat>(u32));
+
+      dstExp &= dstEnc.ExpMask;
+    }
+  }
+
+  auto dst = sign | (dstExp << dstEnc.ManBits) | dstMan;
+
+  union {
+    float f32;
+    __amd_fp16_storage_t fp16[2];
+    __amd_bf16_storage_t bf16[2];
+    uint32_t u32;
+  } u;
+  u.u32 = dst;
+  if constexpr (std::is_same<T, float>())
+    return u.f32;
+  else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+    return u.fp16[0];
+  else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+    return u.bf16[0];
+  else
+    __builtin_trap();
+}
+
+template <typename T, Encoding E, bool sat>
+__OCP_FP_HOST_DEVICE_STATIC__ uint32_t from_float_sr(T f, uint32_t seed, int8_t scale_exp) {
+  // We do not support bf16/fp16 <-> float
+  static_assert(E != Encoding::IEEE754 && E != Encoding::E5M10 && E != Encoding::E8M7);
+  static_assert(sizeof(__amd_fp16_storage_t[2]) == sizeof(float));
+  static_assert(sizeof(__amd_bf16_storage_t[2]) == sizeof(float));
+  union {
+    float f32;
+    __amd_fp16_storage_t fp16[2];
+    __amd_bf16_storage_t bf16[2];
+    uint32_t u32;
+  } u;
+
+  if constexpr (std::is_same<T, float>())
+    u.f32 = f;
+  else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+    u.fp16[0] = f;
+  else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+    u.bf16[0] = f;
+  else
+    __builtin_trap();
+
+  const auto& enc = encodings[(size_t)E];
+  const auto srcE = []() -> Encoding {
+    if constexpr (std::is_same<T, float>())
+      return Encoding::IEEE754;
+    else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+      return Encoding::E5M10;
+    else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+      return Encoding::E8M7;
+    else
+      __builtin_trap();
+  }();
+  const auto& srcEnc = encodings[(size_t)srcE];
+
+  auto srcU32 = u.u32;// (srcE == Encoding::IEEE754) ? U32(f) : (uint32_t)f;
+  auto signBit = signbit<srcE, false>(srcU32);
+  auto sign = signBit << (enc.ExpBits + enc.ManBits);
+
+  if (isnan<srcE, sat>(srcU32) || (enc.MxScale && scale_exp == OCP_SCALE_EXP_NAN))
+    return nan<E, sat>(signBit);
+
+  if (isinf<srcE, sat>(srcU32)) return inf<E, sat>(signBit);
+
+  if (iszero<srcE, sat>(srcU32)) return zero<E, sat>(signBit);
+
+  auto srcMan = mantissa<srcE, false>(srcU32);
+  auto srcExp = exponent<srcE, false>(srcU32);
+  if (enc.MxScale) {
+    if (issubnorm<srcE, false>(srcU32)) {
+      auto leadbit = (uint32_t)1 << srcEnc.ManBits;
+      while ((srcMan & leadbit) == 0) {
+        srcMan <<= 1;
+        srcExp -= 1;
+      }
+
+      srcMan &= srcEnc.ManMask;
+    }
+
+    srcExp -= scale_exp;
+  }
+
+  auto exp = srcExp;
+  auto man = srcMan;
+  bool subnorm = false;
+
+  if (exp > enc.MaxExp) {
+    return inf<E, sat>(signBit);
+  } else if (exp >= enc.MinExp) {
+    man = srcMan;
+  } else if (exp < enc.MinExp) {
+    subnorm = true;
+    exp = 0;
+
+    auto diff = (uint32_t)(enc.MinExp - srcExp);
+    if (diff >= 32) {
+      man = 0;
+      srcMan = 0;
+    } else {
+      srcMan |= (uint32_t)1 << srcEnc.ManBits;
+      srcMan >>= diff;
+    }
+
+    man = srcMan;
+  }
+
+  // Align random value to be one past the kept mant bit
+  size_t sr_shift = (32 - srcEnc.ManBits) + enc.ManBits;
+
+  // For stochastic-rounding we add the aligned random value to the
+  // mantissa and then truncate (RTZ).
+  man += seed >> sr_shift;
+
+  // Increment exponent when mantissa overflows due to rounding
+  if (man >= (uint32_t)1 << srcEnc.ManBits) ++exp;
+  man >>= (srcEnc.ManBits - enc.ManBits);
+  man &= enc.ManMask;
+
+  if (exp > enc.MaxExp) return inf<E, sat>(signBit);
+
+  auto biasedExp = (uint32_t)exp;
+  if (!subnorm) biasedExp = (uint32_t)(exp + enc.ExpBias);
+  biasedExp &= enc.ExpMask;
+
+  auto val = sign | biasedExp << enc.ManBits | man;
+  if (inrange<E, sat>(val))
+    return val;
+  else if (man == 0 && exp == 0)
+    return zero<E, sat>(signBit);
+  else
+    return inf<E, sat>(signBit);
+}
+
+
+template <typename T, Encoding E, bool sat>
+__OCP_FP_HOST_DEVICE_STATIC__ uint32_t from_float(T f, int8_t scale_exp) {
+  // We do not support bf16/fp16 <-> float
+  static_assert(E != Encoding::IEEE754 && E != Encoding::E5M10 && E != Encoding::E8M7);
+  static_assert(sizeof(__amd_fp16_storage_t[2]) == sizeof(float));
+  static_assert(sizeof(__amd_bf16_storage_t[2]) == sizeof(float));
+  union {
+    float f32;
+    __amd_fp16_storage_t fp16[2];
+    __amd_bf16_storage_t bf16[2];
+    uint32_t u32;
+  } u;
+
+  if constexpr (std::is_same<T, float>())
+    u.f32 = f;
+  else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+    u.fp16[0] = f;
+  else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+    u.bf16[0] = f;
+  else
+    __builtin_trap();
+
+  const auto& enc = encodings[(size_t)E];
+  const auto srcE = []() -> Encoding {
+    if constexpr (std::is_same<T, float>())
+      return Encoding::IEEE754;
+    else if constexpr (std::is_same<T, __amd_fp16_storage_t>())
+      return Encoding::E5M10;
+    else if constexpr (std::is_same<T, __amd_bf16_storage_t>())
+      return Encoding::E8M7;
+    else
+      __builtin_trap();
+  }();
+  const auto& srcEnc = encodings[(size_t)srcE];
+
+  auto srcU32 = u.u32; // (srcE == Encoding::IEEE754) ? U32(f) : (uint32_t)f;
+  auto signBit = signbit<srcE, false>(srcU32);
+  auto sign = signBit << (enc.ExpBits + enc.ManBits);
+
+  if (isnan<srcE, sat>(srcU32) || (enc.MxScale && scale_exp == OCP_SCALE_EXP_NAN))
+    return nan<E, sat>(signBit);
+
+  if (isinf<srcE, sat>(srcU32)) return inf<E, sat>(signBit);
+
+  if (iszero<srcE, sat>(srcU32)) return zero<E, sat>(signBit);
+
+  auto srcMan = mantissa<srcE, false>(srcU32);
+  auto srcExp = exponent<srcE, false>(srcU32);
+  if (enc.MxScale) {
+    if (issubnorm<srcE, false>(srcU32)) {
+      auto leadbit = (uint32_t)1 << srcEnc.ManBits;
+      while ((srcMan & leadbit) == 0) {
+        srcMan <<= 1;
+        srcExp -= 1;
+      }
+
+      srcMan &= srcEnc.ManMask;
+    }
+
+    srcExp -= scale_exp;
+  }
+
+  auto exp = srcExp;
+  auto man = srcMan;
+  uint32_t stickyBits = 0;
+  bool subnorm = false;
+
+  if (exp > enc.MaxExp) {
+    return inf<E, sat>(signBit);
+  } else if (exp >= enc.MinExp) {
+    man >>= srcEnc.ManBits - enc.ManBits;
+  } else if (exp < enc.MinExp) {
+    subnorm = true;
+    exp = 0;
+
+    auto diff = (uint32_t)(enc.MinExp - srcExp);
+    if (diff >= 32) {
+      man = 0;
+      srcMan = 0;
+    } else {
+      srcMan |= (uint32_t)1 << srcEnc.ManBits;
+      stickyBits = srcMan & (((uint32_t)1 << diff) - (uint32_t)1);
+      srcMan >>= diff;
+
+      man = srcMan;
+      man >>= srcEnc.ManBits - enc.ManBits;
+      man &= enc.ManMask;
+    }
+  }
+
+  auto roundBitShift = srcEnc.ManBits - (enc.ManBits + 1);
+  auto roundBit = ((srcMan >> roundBitShift) & 1) != 0;
+  stickyBits |= srcMan & (((uint32_t)1 << roundBitShift) - 1);
+  auto odd = (man & 1) != 0;
+
+  if ((roundBit && stickyBits != 0) || (roundBit && odd)) {
+    ++man;
+    if ((man & ((uint32_t)1 << enc.ManBits)) != 0) ++exp;
+    man &= enc.ManMask;
+  }
+
+  if (exp > enc.MaxExp) return inf<E, sat>(signBit);
+
+  auto biasedExp = (uint32_t)exp;
+  if (!subnorm) biasedExp = (uint32_t)(exp + enc.ExpBias);
+  biasedExp &= enc.ExpMask;
+
+  auto val = sign | biasedExp << enc.ManBits | man;
+  if (inrange<E, sat>(val))
+    return val;
+  else if (man == 0 && exp == 0)
+    return zero<E, sat>(signBit);
+  else
+    return inf<E, sat>(signBit);
+}
+
+template <typename InType, typename OutType, typename float_base_t, Encoding in_encode,
+          Encoding out_encode, bool sr = false>
+__OCP_FP_HOST_DEVICE_STATIC__ OutType fp6_cvt_packedx32(InType in, int8_t scale = 0,
+                                                        uint32_t seed = 0) {
+  // This is tightly coupled with the definitions of the amd_ocp_types
+  constexpr bool in_float = std::is_same<InType, __amd_floatx32_storage_t>::value ||
+      std::is_same<InType, __amd_fp16x32_storage_t>::value ||
+      std::is_same<InType, __amd_bf16x32_storage_t>::value;
+  constexpr bool out_float = std::is_same<OutType, __amd_floatx32_storage_t>::value ||
+      std::is_same<OutType, __amd_fp16x32_storage_t>::value ||
+      std::is_same<OutType, __amd_bf16x32_storage_t>::value;
+  using other_type = std::conditional<in_float, OutType, InType>::type;
+
+  struct fp6x32_packed {
+    uint8_t val1 : 6;
+    uint8_t val2 : 6;
+    uint8_t val3 : 6;
+    uint8_t val4 : 6;
+    uint8_t val5 : 6;
+    uint8_t val6 : 6;
+    uint8_t val7 : 6;
+    uint8_t val8 : 6;
+    uint8_t val9 : 6;
+    uint8_t val10 : 6;
+    uint8_t val11 : 6;
+    uint8_t val12 : 6;
+    uint8_t val13 : 6;
+    uint8_t val14 : 6;
+    uint8_t val15 : 6;
+    uint8_t val16 : 6;
+    uint8_t val17 : 6;
+    uint8_t val18 : 6;
+    uint8_t val19 : 6;
+    uint8_t val20 : 6;
+    uint8_t val21 : 6;
+    uint8_t val22 : 6;
+    uint8_t val23 : 6;
+    uint8_t val24 : 6;
+    uint8_t val25 : 6;
+    uint8_t val26 : 6;
+    uint8_t val27 : 6;
+    uint8_t val28 : 6;
+    uint8_t val29 : 6;
+    uint8_t val30 : 6;
+    uint8_t val31 : 6;
+    uint8_t val32 : 6;
+    unsigned long long padded;
+  } __attribute__((packed));
+
+  static_assert(sizeof(other_type) == sizeof(fp6x32_packed));
+  union {
+    other_type o;
+    fp6x32_packed fp6;
+  } u;
+
+  // TODO maybe make it simpler
+  if constexpr (in_float) {
+    if constexpr (sr) {
+      u.fp6.val1 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[0], seed, scale));
+      u.fp6.val2 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[1], seed, scale));
+      u.fp6.val3 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[2], seed, scale));
+      u.fp6.val4 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[3], seed, scale));
+      u.fp6.val5 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[4], seed, scale));
+      u.fp6.val6 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[5], seed, scale));
+      u.fp6.val7 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[6], seed, scale));
+      u.fp6.val8 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[7], seed, scale));
+      u.fp6.val9 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[8], seed, scale));
+      u.fp6.val10 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[9], seed, scale));
+      u.fp6.val11 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[10], seed, scale));
+      u.fp6.val12 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[11], seed, scale));
+      u.fp6.val13 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[12], seed, scale));
+      u.fp6.val14 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[13], seed, scale));
+      u.fp6.val15 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[14], seed, scale));
+      u.fp6.val16 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[15], seed, scale));
+      u.fp6.val17 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[16], seed, scale));
+      u.fp6.val18 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[17], seed, scale));
+      u.fp6.val19 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[18], seed, scale));
+      u.fp6.val20 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[19], seed, scale));
+      u.fp6.val21 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[20], seed, scale));
+      u.fp6.val22 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[21], seed, scale));
+      u.fp6.val23 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[22], seed, scale));
+      u.fp6.val24 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[23], seed, scale));
+      u.fp6.val25 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[24], seed, scale));
+      u.fp6.val26 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[25], seed, scale));
+      u.fp6.val27 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[26], seed, scale));
+      u.fp6.val28 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[27], seed, scale));
+      u.fp6.val29 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[28], seed, scale));
+      u.fp6.val30 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[29], seed, scale));
+      u.fp6.val31 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[30], seed, scale));
+      u.fp6.val32 =
+          static_cast<uint8_t>(from_float_sr<float_base_t, out_encode, true>(in[31], seed, scale));
+    } else {
+      u.fp6.val1 = from_float<float_base_t, out_encode, true>(in[0], scale);
+      u.fp6.val2 = from_float<float_base_t, out_encode, true>(in[1], scale);
+      u.fp6.val3 = from_float<float_base_t, out_encode, true>(in[2], scale);
+      u.fp6.val4 = from_float<float_base_t, out_encode, true>(in[3], scale);
+      u.fp6.val5 = from_float<float_base_t, out_encode, true>(in[4], scale);
+      u.fp6.val6 = from_float<float_base_t, out_encode, true>(in[5], scale);
+      u.fp6.val7 = from_float<float_base_t, out_encode, true>(in[6], scale);
+      u.fp6.val8 = from_float<float_base_t, out_encode, true>(in[7], scale);
+      u.fp6.val9 = from_float<float_base_t, out_encode, true>(in[8], scale);
+      u.fp6.val10 = from_float<float_base_t, out_encode, true>(in[9], scale);
+      u.fp6.val11 = from_float<float_base_t, out_encode, true>(in[10], scale);
+      u.fp6.val12 = from_float<float_base_t, out_encode, true>(in[11], scale);
+      u.fp6.val13 = from_float<float_base_t, out_encode, true>(in[12], scale);
+      u.fp6.val14 = from_float<float_base_t, out_encode, true>(in[13], scale);
+      u.fp6.val15 = from_float<float_base_t, out_encode, true>(in[14], scale);
+      u.fp6.val16 = from_float<float_base_t, out_encode, true>(in[15], scale);
+      u.fp6.val17 = from_float<float_base_t, out_encode, true>(in[16], scale);
+      u.fp6.val18 = from_float<float_base_t, out_encode, true>(in[17], scale);
+      u.fp6.val19 = from_float<float_base_t, out_encode, true>(in[18], scale);
+      u.fp6.val20 = from_float<float_base_t, out_encode, true>(in[19], scale);
+      u.fp6.val21 = from_float<float_base_t, out_encode, true>(in[20], scale);
+      u.fp6.val22 = from_float<float_base_t, out_encode, true>(in[21], scale);
+      u.fp6.val23 = from_float<float_base_t, out_encode, true>(in[22], scale);
+      u.fp6.val24 = from_float<float_base_t, out_encode, true>(in[23], scale);
+      u.fp6.val25 = from_float<float_base_t, out_encode, true>(in[24], scale);
+      u.fp6.val26 = from_float<float_base_t, out_encode, true>(in[25], scale);
+      u.fp6.val27 = from_float<float_base_t, out_encode, true>(in[26], scale);
+      u.fp6.val28 = from_float<float_base_t, out_encode, true>(in[27], scale);
+      u.fp6.val29 = from_float<float_base_t, out_encode, true>(in[28], scale);
+      u.fp6.val30 = from_float<float_base_t, out_encode, true>(in[29], scale);
+      u.fp6.val31 = from_float<float_base_t, out_encode, true>(in[30], scale);
+      u.fp6.val32 = from_float<float_base_t, out_encode, true>(in[31], scale);
+    }
+    return u.o;
+  } else {
+    OutType ret;
+    u.o = in;
+    ret[0] = to_float<float_base_t, in_encode, true>(u.fp6.val1, scale);
+    ret[1] = to_float<float_base_t, in_encode, true>(u.fp6.val2, scale);
+    ret[2] = to_float<float_base_t, in_encode, true>(u.fp6.val3, scale);
+    ret[3] = to_float<float_base_t, in_encode, true>(u.fp6.val4, scale);
+    ret[4] = to_float<float_base_t, in_encode, true>(u.fp6.val5, scale);
+    ret[5] = to_float<float_base_t, in_encode, true>(u.fp6.val6, scale);
+    ret[6] = to_float<float_base_t, in_encode, true>(u.fp6.val7, scale);
+    ret[7] = to_float<float_base_t, in_encode, true>(u.fp6.val8, scale);
+    ret[8] = to_float<float_base_t, in_encode, true>(u.fp6.val9, scale);
+    ret[9] = to_float<float_base_t, in_encode, true>(u.fp6.val10, scale);
+    ret[10] = to_float<float_base_t, in_encode, true>(u.fp6.val11, scale);
+    ret[11] = to_float<float_base_t, in_encode, true>(u.fp6.val12, scale);
+    ret[12] = to_float<float_base_t, in_encode, true>(u.fp6.val13, scale);
+    ret[13] = to_float<float_base_t, in_encode, true>(u.fp6.val14, scale);
+    ret[14] = to_float<float_base_t, in_encode, true>(u.fp6.val15, scale);
+    ret[15] = to_float<float_base_t, in_encode, true>(u.fp6.val16, scale);
+    ret[16] = to_float<float_base_t, in_encode, true>(u.fp6.val17, scale);
+    ret[17] = to_float<float_base_t, in_encode, true>(u.fp6.val18, scale);
+    ret[18] = to_float<float_base_t, in_encode, true>(u.fp6.val19, scale);
+    ret[19] = to_float<float_base_t, in_encode, true>(u.fp6.val20, scale);
+    ret[20] = to_float<float_base_t, in_encode, true>(u.fp6.val21, scale);
+    ret[21] = to_float<float_base_t, in_encode, true>(u.fp6.val22, scale);
+    ret[22] = to_float<float_base_t, in_encode, true>(u.fp6.val23, scale);
+    ret[23] = to_float<float_base_t, in_encode, true>(u.fp6.val24, scale);
+    ret[24] = to_float<float_base_t, in_encode, true>(u.fp6.val25, scale);
+    ret[25] = to_float<float_base_t, in_encode, true>(u.fp6.val26, scale);
+    ret[26] = to_float<float_base_t, in_encode, true>(u.fp6.val27, scale);
+    ret[27] = to_float<float_base_t, in_encode, true>(u.fp6.val28, scale);
+    ret[28] = to_float<float_base_t, in_encode, true>(u.fp6.val29, scale);
+    ret[29] = to_float<float_base_t, in_encode, true>(u.fp6.val30, scale);
+    ret[30] = to_float<float_base_t, in_encode, true>(u.fp6.val31, scale);
+    ret[31] = to_float<float_base_t, in_encode, true>(u.fp6.val32, scale);
+    return ret;
+  }
+}
+}  // namespace fcbx

projects/clr/hipamd/include/hip/amd_detail/amd_hip_ocp_types.h

@@ -0,0 +1,90 @@
+/*
+Copyright © Advanced Micro Devices, Inc., or its affiliates.
+ 
+SPDX-License-Identifier: MIT
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#define __OCP_FP_HOST__ __host__
+#define __OCP_FP_DEVICE__ __device__
+#define __OCP_FP_HOST_DEVICE__ __OCP_FP_HOST__ __OCP_FP_DEVICE__
+#define __OCP_FP_DEVICE_STATIC__ __OCP_FP_DEVICE__ static __inline__ __attribute__((always_inline))
+#define __OCP_FP_HOST_DEVICE_STATIC__ __OCP_FP_HOST_DEVICE__ static
+
+static_assert(sizeof(unsigned int) == 4);
+static_assert(sizeof(float) == 4);
+static_assert(sizeof(unsigned short) == 2);
+
+#if (defined(__clang__) && (__clang_major__ > 17) && defined(__HIP__)) ||                          \
+    (defined(__GNUC__) && (__GNUC__ > 13))
+static_assert(sizeof(__bf16) == 2);
+static_assert(sizeof(_Float16) == 2);
+#endif
+
+// Although we do have some abstractions of half and bfloat16, since this will be a standalone
+// header which will act as a base abstraction, and will be maintained in the future, it makes sense
+// to keep these vector types separate from existing implementations. We can add conversion
+// functions in a different header using these functions.
+typedef uint8_t  __amd_fp8_storage_t;
+typedef uint16_t __amd_fp8x2_storage_t;
+typedef uint8_t  __amd_fp4x2_storage_t;
+typedef uint32_t __amd_fp4x8_storage_t;
+typedef __bf16   __amd_bf16_storage_t;
+typedef _Float16 __amd_fp16_storage_t;
+typedef int8_t  __amd_scale_t;
+
+#if defined(__clang__) && (__clang_major__ > 17) && defined(__HIP__)
+typedef unsigned int __attribute__((ext_vector_type(2))) __amd_uintx2_storage_t;
+typedef uint8_t __attribute__((ext_vector_type(8))) __amd_fp8x8_storage_t;
+typedef __bf16 __attribute__((ext_vector_type(2))) __amd_bf16x2_storage_t;
+typedef __bf16 __attribute__((ext_vector_type(8))) __amd_bf16x8_storage_t;
+typedef __bf16 __attribute__((ext_vector_type(32))) __amd_bf16x32_storage_t;
+typedef float __attribute__((ext_vector_type(2))) __amd_floatx2_storage_t;
+typedef float __attribute__((ext_vector_type(8))) __amd_floatx8_storage_t;
+typedef float __attribute__((ext_vector_type(16))) __amd_floatx16_storage_t;
+typedef float __attribute__((ext_vector_type(32))) __amd_floatx32_storage_t;
+typedef _Float16 __attribute__((ext_vector_type(2))) __amd_fp16x2_storage_t;
+typedef _Float16 __attribute__((ext_vector_type(8))) __amd_fp16x8_storage_t;
+typedef _Float16 __attribute__((ext_vector_type(32))) __amd_fp16x32_storage_t;
+typedef uint32_t __attribute__((ext_vector_type(6))) __amd_fp6x32_storage_t;
+typedef short __attribute__((ext_vector_type(2))) __amd_shortx2_storage_t;
+#elif defined(__GNUC__) && (__GNUC__ > 13)
+/* GCC expects vector size in bytes */
+typedef unsigned int __attribute__((vector_size(8))) __amd_uintx2_storage_t;
+typedef uint8_t __attribute__((vector_size(8))) __amd_fp8x8_storage_t;
+typedef __bf16 __attribute__((vector_size(4))) __amd_bf16x2_storage_t;
+typedef __bf16 __attribute__((vector_size(16))) __amd_bf16x8_storage_t;
+typedef __bf16 __attribute__((vector_size(64))) __amd_bf16x32_storage_t;
+typedef float __attribute__((vector_size(8))) __amd_floatx2_storage_t;
+typedef float __attribute__((vector_size(32))) __amd_floatx8_storage_t;
+typedef float __attribute__((vector_size(64))) __amd_floatx16_storage_t;
+typedef float __attribute__((vector_size(128))) __amd_floatx32_storage_t;
+typedef _Float16 __attribute__((vector_size(4))) __amd_fp16x2_storage_t;
+typedef _Float16 __attribute__((vector_size(16))) __amd_fp16x8_storage_t;
+typedef _Float16 __attribute__((vector_size(64))) __amd_fp16x32_storage_t;
+typedef uint32_t __attribute__((vector_size(24))) __amd_fp6x32_storage_t;
+typedef short __attribute__((vector_size(4))) __amd_shortx2_storage_t;
+#else
+#error "Only supported by HIPCC or GCC >= 13."
+#endif
+
+static_assert(sizeof(__amd_uintx2_storage_t) == sizeof(__amd_fp8x8_storage_t));

projects/clr/hipamd/include/hip/amd_detail/amd_hip_runtime.h

@@ -0,0 +1,393 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/hip_runtime.h
+ *  @brief Contains definitions of APIs for HIP runtime.
+ */
+
+//#pragma once
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
+
+#include <hip/amd_detail/amd_hip_common.h>
+
+#if !defined(__HIPCC_RTC__)
+#ifdef __cplusplus
+#include <cstddef>
+#else
+#include <stddef.h>
+#endif  // __cplusplus
+#endif  // !defined(__HIPCC_RTC__)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Query the installed library build name.
+ *
+ * This function can be used even when the library is not initialized.
+ *
+ * @returns Returns a string describing the build version of the library.  The
+ * string is owned by the library.
+ */
+const char* amd_dbgapi_get_build_name();
+
+/**
+ * @brief Query the installed library git hash.
+ *
+ * This function can be used even when the library is not initialized.
+ *
+ * @returns Returns git hash of the library.
+ */
+const char* amd_dbgapi_get_git_hash();
+
+/**
+ * @brief Query the installed library build ID.
+ *
+ * This function can be used even when the library is not initialized.
+ *
+ * @returns Returns build ID of the library.
+ */
+size_t amd_dbgapi_get_build_id();
+
+#ifdef __cplusplus
+} /* extern "c" */
+#endif
+
+//---
+// Top part of file can be compiled with any compiler
+
+#if !defined(__HIPCC_RTC__)
+#ifdef __cplusplus
+#include <cmath>
+#include <cstdint>
+#include <tuple>
+#else
+#include <math.h>
+#include <stdint.h>
+#endif // __cplusplus
+#endif // !defined(__HIPCC_RTC__)
+
+#if __HIP_CLANG_ONLY__
+
+#if !defined(__align__)
+#define __align__(x) __attribute__((aligned(x)))
+#endif
+
+#define CUDA_SUCCESS hipSuccess
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_runtime_api.h>
+#include <hip/amd_detail/amd_hip_atomic.h>
+#include <hip/amd_detail/amd_device_functions.h>
+#include <hip/amd_detail/amd_surface_functions.h>
+#include <hip/amd_detail/texture_fetch_functions.h>
+#include <hip/amd_detail/texture_indirect_functions.h>
+extern int HIP_TRACE_API;
+#endif // !defined(__HIPCC_RTC__)
+
+#ifdef __cplusplus
+#include <hip/amd_detail/hip_ldg.h>
+#endif
+
+#include <hip/amd_detail/host_defines.h>
+
+// TODO-HCC remove old definitions ; ~1602 hcc supports __HCC_ACCELERATOR__ define.
+#if defined(__KALMAR_ACCELERATOR__) && !defined(__HCC_ACCELERATOR__)
+#define __HCC_ACCELERATOR__ __KALMAR_ACCELERATOR__
+#endif
+
+// Feature tests:
+#if (defined(__HCC_ACCELERATOR__) && (__HCC_ACCELERATOR__ != 0)) || __HIP_DEVICE_COMPILE__
+// Device compile and not host compile:
+
+// 32-bit Atomics:
+#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (1)
+#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (1)
+#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (1)
+#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (1)
+#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (1)
+
+// 64-bit Atomics:
+#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (1)
+#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (1)
+
+// Doubles
+#define __HIP_ARCH_HAS_DOUBLES__ (1)
+
+// warp cross-lane operations:
+#define __HIP_ARCH_HAS_WARP_VOTE__ (1)
+#define __HIP_ARCH_HAS_WARP_BALLOT__ (1)
+#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (1)
+#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (0)
+
+// sync
+#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (1)
+#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (0)
+
+// misc
+#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (0)
+#define __HIP_ARCH_HAS_3DGRID__ (1)
+#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (0)
+
+#endif /* Device feature flags */
+
+
+#define launch_bounds_impl0(requiredMaxThreadsPerBlock)                                            \
+    __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock)))
+#define launch_bounds_impl1(requiredMaxThreadsPerBlock, minBlocksPerMultiprocessor)                \
+    __attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock),                     \
+                   amdgpu_waves_per_eu(minBlocksPerMultiprocessor)))
+#define select_impl_(_1, _2, impl_, ...) impl_
+#define __launch_bounds__(...)                                                                     \
+  select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0, )(__VA_ARGS__)
+
+#if !defined(__HIPCC_RTC__)
+__host__ inline void* __get_dynamicgroupbaseptr() { return nullptr; }
+#endif // !defined(__HIPCC_RTC__)
+
+// End doxygen API:
+/**
+ *   @}
+ */
+
+//
+// hip-clang functions
+//
+#if !defined(__HIPCC_RTC__)
+#define HIP_KERNEL_NAME(...) __VA_ARGS__
+#define HIP_SYMBOL(X) X
+
+typedef int hipLaunchParm;
+
+template <std::size_t n, typename... Ts,
+          typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
+void pArgs(const std::tuple<Ts...>&, void*) {}
+
+template <std::size_t n, typename... Ts,
+          typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
+void pArgs(const std::tuple<Ts...>& formals, void** _vargs) {
+    using T = typename std::tuple_element<n, std::tuple<Ts...> >::type;
+
+    static_assert(!std::is_reference<T>{},
+                  "A __global__ function cannot have a reference as one of its "
+                  "arguments.");
+#if defined(HIP_STRICT)
+    static_assert(std::is_trivially_copyable<T>{},
+                  "Only TriviallyCopyable types can be arguments to a __global__ "
+                  "function");
+#endif
+    _vargs[n] = const_cast<void*>(reinterpret_cast<const void*>(&std::get<n>(formals)));
+    return pArgs<n + 1>(formals, _vargs);
+}
+
+template <typename... Formals, typename... Actuals>
+std::tuple<Formals...> validateArgsCountType(void (*kernel)(Formals...), std::tuple<Actuals...>(actuals)) {
+    static_assert(sizeof...(Formals) == sizeof...(Actuals), "Argument Count Mismatch");
+    std::tuple<Formals...> to_formals{std::move(actuals)};
+    return to_formals;
+}
+
+#if defined(HIP_TEMPLATE_KERNEL_LAUNCH)
+template <typename... Args, typename F = void (*)(Args...)>
+void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
+                        std::uint32_t sharedMemBytes, hipStream_t stream, Args... args) {
+    constexpr size_t count = sizeof...(Args);
+    auto tup_ = std::tuple<Args...>{args...};
+    auto tup = validateArgsCountType(kernel, tup_);
+    void* _Args[count];
+    pArgs<0>(tup, _Args);
+
+    auto k = reinterpret_cast<void*>(kernel);
+    hipLaunchKernel(k, numBlocks, dimBlocks, _Args, sharedMemBytes, stream);
+}
+#else
+#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...)  \
+    do {                                                                                           \
+        kernelName<<<(numBlocks), (numThreads), (memPerBlock), (streamId)>>>(__VA_ARGS__);         \
+    } while (0)
+
+#define hipLaunchKernelGGL(kernelName, ...)  hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__)
+#endif
+
+#include <hip/hip_runtime_api.h>
+#endif // !defined(__HIPCC_RTC__)
+
+#if defined(__HIPCC_RTC__)
+typedef struct dim3 {
+    __hip_uint32_t x;  ///< x
+    __hip_uint32_t y;  ///< y
+    __hip_uint32_t z;  ///< z
+#ifdef __cplusplus
+    constexpr __device__ dim3(__hip_uint32_t _x = 1, __hip_uint32_t _y = 1, __hip_uint32_t _z = 1)
+            : x(_x), y(_y), z(_z){};
+#endif
+} dim3;
+#endif // !defined(__HIPCC_RTC__)
+
+#pragma push_macro("__DEVICE__")
+#define __DEVICE__ static __device__ __forceinline__
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(unsigned int);
+__DEVICE__ unsigned int __hip_get_thread_idx_x() { return __ockl_get_local_id(0); }
+__DEVICE__ unsigned int __hip_get_thread_idx_y() { return __ockl_get_local_id(1); }
+__DEVICE__ unsigned int __hip_get_thread_idx_z() { return __ockl_get_local_id(2); }
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(unsigned int);
+__DEVICE__ unsigned int __hip_get_block_idx_x() { return __ockl_get_group_id(0); }
+__DEVICE__ unsigned int __hip_get_block_idx_y() { return __ockl_get_group_id(1); }
+__DEVICE__ unsigned int __hip_get_block_idx_z() { return __ockl_get_group_id(2); }
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(unsigned int);
+__DEVICE__ unsigned int __hip_get_block_dim_x() { return __ockl_get_local_size(0); }
+__DEVICE__ unsigned int __hip_get_block_dim_y() { return __ockl_get_local_size(1); }
+__DEVICE__ unsigned int __hip_get_block_dim_z() { return __ockl_get_local_size(2); }
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(unsigned int);
+__DEVICE__ unsigned int __hip_get_grid_dim_x() { return __ockl_get_num_groups(0); }
+__DEVICE__ unsigned int __hip_get_grid_dim_y() { return __ockl_get_num_groups(1); }
+__DEVICE__ unsigned int __hip_get_grid_dim_z() { return __ockl_get_num_groups(2); }
+
+#define __HIP_DEVICE_BUILTIN(DIMENSION, FUNCTION)               \
+  __declspec(property(get = __get_##DIMENSION)) unsigned int DIMENSION; \
+  __DEVICE__ unsigned int __get_##DIMENSION(void) {                     \
+    return FUNCTION;                                            \
+  }
+
+struct __hip_builtin_threadIdx_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_thread_idx_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_thread_idx_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_thread_idx_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+struct __hip_builtin_blockIdx_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_block_idx_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_block_idx_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_block_idx_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+struct __hip_builtin_blockDim_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_block_dim_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_block_dim_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_block_dim_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+struct __hip_builtin_gridDim_t {
+  __HIP_DEVICE_BUILTIN(x,__hip_get_grid_dim_x());
+  __HIP_DEVICE_BUILTIN(y,__hip_get_grid_dim_y());
+  __HIP_DEVICE_BUILTIN(z,__hip_get_grid_dim_z());
+#ifdef __cplusplus
+  __device__ operator dim3() const { return dim3(x, y, z); }
+#endif
+};
+
+#undef __HIP_DEVICE_BUILTIN
+#pragma pop_macro("__DEVICE__")
+
+extern const __device__ __attribute__((weak)) __hip_builtin_threadIdx_t threadIdx;
+extern const __device__ __attribute__((weak)) __hip_builtin_blockIdx_t blockIdx;
+extern const __device__ __attribute__((weak)) __hip_builtin_blockDim_t blockDim;
+extern const __device__ __attribute__((weak)) __hip_builtin_gridDim_t gridDim;
+
+#define hipThreadIdx_x threadIdx.x
+#define hipThreadIdx_y threadIdx.y
+#define hipThreadIdx_z threadIdx.z
+
+#define hipBlockIdx_x blockIdx.x
+#define hipBlockIdx_y blockIdx.y
+#define hipBlockIdx_z blockIdx.z
+
+#define hipBlockDim_x blockDim.x
+#define hipBlockDim_y blockDim.y
+#define hipBlockDim_z blockDim.z
+
+#define hipGridDim_x gridDim.x
+#define hipGridDim_y gridDim.y
+#define hipGridDim_z gridDim.z
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/amd_detail/amd_math_functions.h>
+#endif
+
+#if __HIP_HCC_COMPAT_MODE__
+// Define HCC work item functions in terms of HIP builtin variables.
+#pragma push_macro("__DEFINE_HCC_FUNC")
+#define __DEFINE_HCC_FUNC(hc_fun,hip_var) \
+inline __device__ __attribute__((always_inline)) unsigned int hc_get_##hc_fun(unsigned int i) { \
+  if (i==0) \
+    return hip_var.x; \
+  else if(i==1) \
+    return hip_var.y; \
+  else \
+    return hip_var.z; \
+}
+
+__DEFINE_HCC_FUNC(workitem_id, threadIdx)
+__DEFINE_HCC_FUNC(group_id, blockIdx)
+__DEFINE_HCC_FUNC(group_size, blockDim)
+__DEFINE_HCC_FUNC(num_groups, gridDim)
+#pragma pop_macro("__DEFINE_HCC_FUNC")
+
+extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_id(unsigned int);
+inline __device__ __attribute__((always_inline)) unsigned int
+hc_get_workitem_absolute_id(int dim)
+{
+  return (unsigned int)__ockl_get_global_id(dim);
+}
+
+#endif
+
+#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+#if !defined(__HIPCC_RTC__)
+// Support std::complex.
+#if !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
+#pragma push_macro("__CUDA__")
+#define __CUDA__
+#include <__clang_cuda_math_forward_declares.h>
+#include <__clang_cuda_complex_builtins.h>
+// Workaround for using libc++ with HIP-Clang.
+// The following headers requires clang include path before standard C++ include path.
+// However libc++ include path requires to be before clang include path.
+// To workaround this, we pass -isystem with the parent directory of clang include
+// path instead of the clang include path itself.
+#include <include/cuda_wrappers/algorithm>
+#include <include/cuda_wrappers/complex>
+#include <include/cuda_wrappers/new>
+#undef __CUDA__
+#pragma pop_macro("__CUDA__")
+#endif // !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
+#endif // !defined(__HIPCC_RTC__)
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+#endif // __HIP_CLANG_ONLY__
+
+#endif  // HIP_AMD_DETAIL_RUNTIME_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_runtime_pt_api.h

@@ -0,0 +1,202 @@
+/*
+Copyright (c) 2022 - Present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#ifndef HIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H
+#define HIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H
+
+#if defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_PLATFORM_NVIDIA__)
+
+/// hipStreamPerThread implementation
+#if defined(HIP_API_PER_THREAD_DEFAULT_STREAM)
+    #define __HIP_STREAM_PER_THREAD
+    #define __HIP_API_SPT(api) api ## _spt
+#else
+    #define __HIP_API_SPT(api) api
+#endif
+
+#if defined(__HIP_STREAM_PER_THREAD)
+    // Memory APIs
+    #define hipMemcpy                     __HIP_API_SPT(hipMemcpy)
+    #define hipMemcpyToSymbol             __HIP_API_SPT(hipMemcpyToSymbol)
+    #define hipMemcpyFromSymbol           __HIP_API_SPT(hipMemcpyFromSymbol)
+    #define hipMemcpy2D                   __HIP_API_SPT(hipMemcpy2D)
+    #define hipMemcpy2DFromArray          __HIP_API_SPT(hipMemcpy2DFromArray)
+    #define hipMemcpy3D                   __HIP_API_SPT(hipMemcpy3D)
+    #define hipMemset                     __HIP_API_SPT(hipMemset)
+    #define hipMemset2D                   __HIP_API_SPT(hipMemset2D)
+    #define hipMemset3D                   __HIP_API_SPT(hipMemset3D)
+    #define hipMemcpyAsync                __HIP_API_SPT(hipMemcpyAsync)
+    #define hipMemset3DAsync              __HIP_API_SPT(hipMemset3DAsync)
+    #define hipMemset2DAsync              __HIP_API_SPT(hipMemset2DAsync)
+    #define hipMemsetAsync                __HIP_API_SPT(hipMemsetAsync)
+    #define hipMemcpy3DAsync              __HIP_API_SPT(hipMemcpy3DAsync)
+    #define hipMemcpy2DAsync              __HIP_API_SPT(hipMemcpy2DAsync)
+    #define hipMemcpyFromSymbolAsync      __HIP_API_SPT(hipMemcpyFromSymbolAsync)
+    #define hipMemcpyToSymbolAsync        __HIP_API_SPT(hipMemcpyToSymbolAsync)
+    #define hipMemcpyFromArray            __HIP_API_SPT(hipMemcpyFromArray)
+    #define hipMemcpy2DToArray            __HIP_API_SPT(hipMemcpy2DToArray)
+    #define hipMemcpy2DFromArrayAsync     __HIP_API_SPT(hipMemcpy2DFromArrayAsync)
+    #define hipMemcpy2DToArrayAsync       __HIP_API_SPT(hipMemcpy2DToArrayAsync)
+
+    // Stream APIs
+    #define hipStreamSynchronize          __HIP_API_SPT(hipStreamSynchronize)
+    #define hipStreamQuery                __HIP_API_SPT(hipStreamQuery)
+    #define hipStreamGetFlags             __HIP_API_SPT(hipStreamGetFlags)
+    #define hipStreamGetPriority          __HIP_API_SPT(hipStreamGetPriority)
+    #define hipStreamWaitEvent            __HIP_API_SPT(hipStreamWaitEvent)
+    #define hipStreamAddCallback          __HIP_API_SPT(hipStreamAddCallback)
+    #define hipLaunchHostFunc             __HIP_API_SPT(hipLaunchHostFunc)
+
+    // Event APIs
+    #define hipEventRecord               __HIP_API_SPT(hipEventRecord)
+
+    // Launch APIs
+    #define hipLaunchKernel               __HIP_API_SPT(hipLaunchKernel)
+    #define hipLaunchCooperativeKernel    __HIP_API_SPT(hipLaunchCooperativeKernel)
+
+    // Graph APIs
+    #define hipGraphLaunch                __HIP_API_SPT(hipGraphLaunch)
+    #define hipStreamBeginCapture         __HIP_API_SPT(hipStreamBeginCapture)
+    #define hipStreamEndCapture           __HIP_API_SPT(hipStreamEndCapture)
+    #define hipStreamIsCapturing          __HIP_API_SPT(hipStreamIsCapturing)
+    #define hipStreamGetCaptureInfo       __HIP_API_SPT(hipStreamGetCaptureInfo)
+    #define hipStreamGetCaptureInfo_v2    __HIP_API_SPT(hipStreamGetCaptureInfo_v2)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+hipError_t hipMemcpy_spt(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind);
+
+hipError_t hipMemcpyToSymbol_spt(const void* symbol, const void* src, size_t sizeBytes,
+                                 size_t offset __dparm(0),
+                                 hipMemcpyKind kind __dparm(hipMemcpyHostToDevice));
+
+hipError_t hipMemcpyFromSymbol_spt(void* dst, const void* symbol,size_t sizeBytes,
+                                   size_t offset __dparm(0),
+                                   hipMemcpyKind kind __dparm(hipMemcpyDeviceToHost));
+
+hipError_t hipMemcpy2D_spt(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
+                        size_t height, hipMemcpyKind kind);
+
+hipError_t hipMemcpy2DFromArray_spt( void* dst, size_t dpitch, hipArray_const_t src, size_t wOffset,
+                        size_t hOffset, size_t width, size_t height, hipMemcpyKind kind);
+
+hipError_t hipMemcpy3D_spt(const struct hipMemcpy3DParms* p);
+
+hipError_t hipMemset_spt(void* dst, int value, size_t sizeBytes);
+
+hipError_t hipMemsetAsync_spt(void* dst, int value, size_t sizeBytes,
+                                            hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemset2D_spt(void* dst, size_t pitch, int value, size_t width, size_t height);
+
+hipError_t hipMemset2DAsync_spt(void* dst, size_t pitch, int value,
+                            size_t width, size_t height,
+                            hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemset3DAsync_spt(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent,
+                                                hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemset3D_spt(hipPitchedPtr pitchedDevPtr, int  value, hipExtent extent );
+
+hipError_t hipMemcpyAsync_spt(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind,
+                          hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpy3DAsync_spt(const hipMemcpy3DParms* p,
+                                hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpy2DAsync_spt(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
+                            size_t height, hipMemcpyKind kind,
+                            hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpyFromSymbolAsync_spt(void* dst, const void* symbol, size_t sizeBytes,
+                                    size_t offset, hipMemcpyKind kind,
+                                    hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpyToSymbolAsync_spt(const void* symbol, const void* src, size_t sizeBytes,
+                                  size_t offset, hipMemcpyKind kind,
+                                  hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpyFromArray_spt(void* dst, hipArray_const_t src, size_t wOffsetSrc, size_t hOffset,
+                                  size_t count, hipMemcpyKind kind);
+
+hipError_t hipMemcpy2DToArray_spt(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src,
+                                  size_t spitch, size_t width, size_t height, hipMemcpyKind kind);
+
+hipError_t hipMemcpy2DFromArrayAsync_spt(void* dst, size_t dpitch, hipArray_const_t src,
+                                  size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height,
+                                  hipMemcpyKind kind,
+                                  hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipMemcpy2DToArrayAsync_spt(hipArray_t dst, size_t wOffset, size_t hOffset, const void* src,
+                                  size_t spitch, size_t width, size_t height, hipMemcpyKind kind,
+                                  hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipStreamQuery_spt(hipStream_t stream);
+
+hipError_t hipStreamSynchronize_spt(hipStream_t stream);
+
+hipError_t hipStreamGetPriority_spt(hipStream_t stream, int* priority);
+
+hipError_t hipStreamWaitEvent_spt(hipStream_t stream, hipEvent_t event, unsigned int flags __dparm(0));
+
+hipError_t hipStreamGetFlags_spt(hipStream_t stream, unsigned int* flags);
+
+hipError_t hipStreamAddCallback_spt(hipStream_t stream, hipStreamCallback_t callback, void* userData,
+                                unsigned int flags);
+
+hipError_t hipEventRecord_spt(hipEvent_t event, hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipLaunchCooperativeKernel_spt(const void* f,
+                                      dim3 gridDim, dim3 blockDim,
+                                      void **kernelParams, uint32_t sharedMemBytes,
+                                      hipStream_t hStream __dparm(hipStreamPerThread));
+
+hipError_t hipLaunchKernel_spt(const void* function_address,
+                           dim3 numBlocks,
+                           dim3 dimBlocks,
+                           void** args,
+                           size_t sharedMemBytes, hipStream_t stream __dparm(hipStreamPerThread));
+
+hipError_t hipGraphLaunch_spt(hipGraphExec_t graphExec, hipStream_t stream);
+hipError_t hipStreamBeginCapture_spt(hipStream_t stream, hipStreamCaptureMode mode);
+hipError_t hipStreamEndCapture_spt(hipStream_t stream, hipGraph_t* pGraph);
+hipError_t hipStreamIsCapturing_spt(hipStream_t stream, hipStreamCaptureStatus* pCaptureStatus);
+hipError_t hipStreamGetCaptureInfo_spt(hipStream_t stream, hipStreamCaptureStatus* pCaptureStatus,
+                                   unsigned long long* pId);
+hipError_t hipStreamGetCaptureInfo_v2_spt(hipStream_t stream, hipStreamCaptureStatus* captureStatus_out,
+                                      unsigned long long* id_out, hipGraph_t* graph_out,
+                                      const hipGraphNode_t** dependencies_out,
+                                      size_t* numDependencies_out);
+hipError_t hipLaunchHostFunc_spt(hipStream_t stream, hipHostFn_t fn, void* userData);
+
+
+#ifdef __cplusplus
+}
+#endif // extern "C"
+
+#endif //defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_PLATFORM_NVIDIA__)
+#endif //HIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H

projects/clr/hipamd/include/hip/amd_detail/amd_hip_unsafe_atomics.h

@@ -0,0 +1,608 @@
+/*
+Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#ifdef __cplusplus
+
+#pragma push_macro("__HIP_ATOMICS_IGNORE_DENORMAL_MODE")
+#if defined(__has_extension) && __has_extension(clang_atomic_attributes)
+#define __HIP_ATOMICS_IGNORE_DENORMAL_MODE [[clang::atomic(ignore_denormal_mode)]]
+#else
+#define __HIP_ATOMICS_IGNORE_DENORMAL_MODE
+#endif
+
+/**
+ * @brief Unsafe floating point rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write floating point atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 4 bytes aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and is not supported.
+ *
+ * @param [in,out] addr Pointer to value to be increment by \p value.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float unsafeAtomicAdd(float* addr, float value) {
+#if defined(__gfx90a__) &&                                                   \
+    __has_builtin(__builtin_amdgcn_is_shared) &&                               \
+    __has_builtin(__builtin_amdgcn_is_private) &&                              \
+    __has_builtin(__builtin_amdgcn_ds_atomic_fadd_f32) &&                      \
+    __has_builtin(__builtin_amdgcn_global_atomic_fadd_f32)
+  if (__builtin_amdgcn_is_shared(
+        (const __attribute__((address_space(0))) void*)addr))
+    return __builtin_amdgcn_ds_atomic_fadd_f32(addr, value);
+  else if (__builtin_amdgcn_is_private(
+              (const __attribute__((address_space(0))) void*)addr)) {
+    float temp = *addr;
+    *addr = temp + value;
+    return temp;
+  }
+  else
+    return __builtin_amdgcn_global_atomic_fadd_f32(addr, value);
+#elif __has_builtin(__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @brief Unsafe floating point rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write floating point atomic max with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if greater.
+ *
+ * @note This operation is currently identical to that performed by
+ * atomicMax and is included for completeness.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float unsafeAtomicMax(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Unsafe floating point rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write floating point atomic min with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if lesser.
+ *
+ * @note This operation is currently identical to that performed by
+ * atomicMin and is included for completeness.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float unsafeAtomicMin(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Unsafe double precision rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write double precision atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 8 byte aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and are not supported.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline double unsafeAtomicAdd(double* addr, double value) {
+#if defined(__gfx90a__) && __has_builtin(__builtin_amdgcn_flat_atomic_fadd_f64)
+  return __builtin_amdgcn_flat_atomic_fadd_f64(addr, value);
+#elif defined (__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @brief Unsafe double precision rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write double precision atomic max with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if greater.
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 8 byte aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and are not supported.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double unsafeAtomicMax(double* addr, double val) {
+#if (defined(__gfx90a__) || defined(__gfx94plus_clr__)) && \
+  __has_builtin(__builtin_amdgcn_flat_atomic_fmax_f64)
+  return __builtin_amdgcn_flat_atomic_fmax_f64(addr, val);
+#else
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+#endif
+}
+
+/**
+ * @brief Unsafe double precision rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write double precision atomic min with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if lesser.
+ *
+ * @note This operation currently only performs different operations for
+ * the gfx90a target. Other devices continue to use safe atomics.
+ *
+ * It can be used to generate code that uses fast hardware floating point atomic
+ * operations which may handle rounding and subnormal values differently than
+ * non-atomic floating point operations.
+ *
+ * The operation is not always safe and can have undefined behavior unless
+ * following condition are met:
+ *
+ * - \p addr is at least 8 byte aligned
+ * - If \p addr is a global segment address, it is in a coarse grain allocation.
+ * Passing in global segment addresses in fine grain allocations will result in
+ * undefined behavior and are not supported.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double unsafeAtomicMin(double* addr, double val) {
+#if (defined(__gfx90a__) || defined(__gfx94plus_clr__)) && \
+  __has_builtin(__builtin_amdgcn_flat_atomic_fmin_f64)
+  return __builtin_amdgcn_flat_atomic_fmin_f64(addr, val);
+#else
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+#endif
+}
+
+/**
+ * @brief Safe floating point rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write floating point atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be increment by \p value.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float safeAtomicAdd(float* addr, float value) {
+#if defined(__gfx908__)                               \
+    || ((defined(__gfx90a__) || defined(__gfx942__) || \
+    defined(__gfx950__)) && !__has_builtin(__hip_atomic_fetch_add))
+  // On gfx908, we can generate unsafe FP32 atomic add that does not follow all
+  // IEEE rules when -munsafe-fp-atomics is passed. Do a CAS loop emulation instead.
+  // On gfx90a, gfx942 and gfx950 if we do not have the __hip_atomic_fetch_add builtin, we
+  // need to force a CAS loop here.
+  float old_val;
+#if __has_builtin(__hip_atomic_load)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else // !__has_builtin(__hip_atomic_load)
+  old_val = __uint_as_float(__atomic_load_n(reinterpret_cast<unsigned int*>(addr), __ATOMIC_RELAXED));
+#endif // __has_builtin(__hip_atomic_load)
+  float expected, temp;
+  do {
+    temp = expected = old_val;
+#if __has_builtin(__hip_atomic_compare_exchange_strong)
+    __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+      __hip_atomic_compare_exchange_strong(addr, &expected, old_val + value, __ATOMIC_RELAXED,
+                                         __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+#else // !__has_builtin(__hip_atomic_compare_exchange_strong)
+    __atomic_compare_exchange_n(addr, &expected, old_val + value, false,
+                                __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+#endif // __has_builtin(__hip_atomic_compare_exchange_strong)
+    old_val = expected;
+  } while (__float_as_uint(temp) != __float_as_uint(old_val));
+  return old_val;
+#elif defined(__gfx90a__)
+  // On gfx90a, with the __hip_atomic_fetch_add builtin, relaxed system-scope
+  // atomics will produce safe CAS loops, but are otherwise not different than
+  // agent-scope atomics. This logic is only applicable for gfx90a, and should
+  // not be assumed on other architectures.
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#elif __has_builtin(__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @brief Safe floating point rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write floating point atomic max with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if greater.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float safeAtomicMax(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Safe floating point rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write floating point atomic min with
+ * device memory scope. The original value at \p addr is returned and
+ * the value at \p addr is replaced by \p val if lesser.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated
+ * @param [in] val Value used to update the value at \p addr.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline float safeAtomicMin(float* addr, float val) {
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    float value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned int *uaddr = (unsigned int *)addr;
+  unsigned int value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __uint_as_float(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __float_as_uint(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __uint_as_float(value);
+  #endif
+}
+
+/**
+ * @brief Safe double precision rmw atomic add.
+ *
+ * Performs a relaxed read-modify-write double precision atomic add with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated to have the original value plus \p value
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be increment by \p value.
+ * @param [in] value Value by \p addr is to be incremented.
+ * @return Original value contained in \p addr.
+ */
+__device__ inline double safeAtomicAdd(double* addr, double value) {
+#if defined(__gfx90a__) &&  __has_builtin(__hip_atomic_fetch_add)
+  // On gfx90a, with the __hip_atomic_fetch_add builtin, relaxed system-scope
+  // atomics will produce safe CAS loops, but are otherwise not different than
+  // agent-scope atomics. This logic is only applicable for gfx90a, and should
+  // not be assumed on other architectures.
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
+  }
+#elif defined(__gfx90a__)
+  // On gfx90a, if we do not have the __hip_atomic_fetch_add builtin, we need to
+  // force a CAS loop here.
+  double old_val;
+#if __has_builtin(__hip_atomic_load)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    old_val = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else // !__has_builtin(__hip_atomic_load)
+  old_val = __longlong_as_double(__atomic_load_n(reinterpret_cast<unsigned long long*>(addr), __ATOMIC_RELAXED));
+#endif // __has_builtin(__hip_atomic_load)
+  double expected, temp;
+  do {
+    temp = expected = old_val;
+#if __has_builtin(__hip_atomic_compare_exchange_strong)
+    __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+      __hip_atomic_compare_exchange_strong(addr, &expected, old_val + value, __ATOMIC_RELAXED,
+                                         __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+#else // !__has_builtin(__hip_atomic_compare_exchange_strong)
+    __atomic_compare_exchange_n(addr, &expected, old_val + value, false,
+                                __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+#endif // __has_builtin(__hip_atomic_compare_exchange_strong)
+    old_val = expected;
+  } while (__double_as_longlong(temp) != __double_as_longlong(old_val));
+  return old_val;
+#else // !defined(__gfx90a__)
+#if __has_builtin(__hip_atomic_fetch_add)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    return __hip_atomic_fetch_add(addr, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+  }
+#else  // !__has_builtin(__hip_atomic_fetch_add)
+  return __atomic_fetch_add(addr, value, __ATOMIC_RELAXED);
+#endif // __has_builtin(__hip_atomic_fetch_add)
+#endif
+}
+
+/**
+ * @brief Safe double precision rmw atomic max.
+ *
+ * Performs a relaxed read-modify-write double precision atomic max with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if greater.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double safeAtomicMax(double* addr, double val) {
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  if (__builtin_amdgcn_is_private(
+          (const __attribute__((address_space(0))) void*)addr)) {
+    double old = *addr;
+    *addr = __builtin_fmax(old, val);
+    return old;
+  } else {
+  #endif
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value < val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) < val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  }
+  #endif
+}
+
+/**
+ * @brief Safe double precision rmw atomic min.
+ *
+ * Performs a relaxed read-modify-write double precision atomic min with
+ * device memory scope. Original value at \p addr is returned and
+ * the value of \p addr is updated with \p val if lesser.
+ *
+ * @note This operation ensures that, on all targets, we produce safe atomics.
+ * This will be the case even when -munsafe-fp-atomics is passed into the compiler.
+ *
+ * @param [in,out] addr Pointer to value to be updated.
+ * @param [in] val Value used to updated the contents at \p addr
+ * @return Original value contained at \p addr.
+ */
+__device__ inline double safeAtomicMin(double* addr, double val) {
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  if (__builtin_amdgcn_is_private(
+           (const __attribute__((address_space(0))) void*)addr)) {
+    double old = *addr;
+    *addr = __builtin_fmin(old, val);
+    return old;
+  } else {
+  #endif
+  #if __has_builtin(__hip_atomic_load) && \
+      __has_builtin(__hip_atomic_compare_exchange_strong)
+  __HIP_ATOMICS_IGNORE_DENORMAL_MODE {
+    double value = __hip_atomic_load(addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    bool done = false;
+    while (!done && value > val) {
+      done = __hip_atomic_compare_exchange_strong(addr, &value, val,
+                 __ATOMIC_RELAXED, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+    }
+    return value;
+  }
+  #else
+  unsigned long long *uaddr = (unsigned long long *)addr;
+  unsigned long long value = __atomic_load_n(uaddr, __ATOMIC_RELAXED);
+  bool done = false;
+  while (!done && __longlong_as_double(value) > val) {
+    done = __atomic_compare_exchange_n(uaddr, &value, __double_as_longlong(val), false,
+               __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+  }
+  return __longlong_as_double(value);
+  #endif
+  #if __has_builtin(__builtin_amdgcn_is_private)
+  }
+  #endif
+}
+
+#pragma pop_macro("__HIP_ATOMICS_IGNORE_DENORMAL_MODE")
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_math_functions.h

@@ -0,0 +1,91 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include "hip_fp16_math_fwd.h"
+#include "amd_hip_vector_types.h"
+#include "math_fwd.h"
+
+#include <hip/amd_detail/host_defines.h>
+
+#include <algorithm>
+// assert.h is only for the host version of assert.
+// The device version of assert is implemented in hip/amd_detail/hip_runtime.h.
+// Users should include hip_runtime.h for the device version of assert.
+#if !__HIP_DEVICE_COMPILE__
+#include <assert.h>
+#endif
+#include <limits.h>
+#include <limits>
+#include <stdint.h>
+#endif // !defined(__HIPCC_RTC__)
+
+#pragma push_macro("__DEVICE__")
+#pragma push_macro("__RETURN_TYPE")
+
+#define __DEVICE__ static __device__
+#define __RETURN_TYPE bool
+
+// DOT FUNCTIONS
+#if defined(__clang__) && defined(__HIP__)
+__DEVICE__
+inline
+int amd_mixed_dot(short2 a, short2 b, int c, bool saturate) {
+  return __ockl_sdot2(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+uint amd_mixed_dot(ushort2 a, ushort2 b, uint c, bool saturate) {
+  return __ockl_udot2(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+int amd_mixed_dot(char4 a, char4 b, int c, bool saturate) {
+  return __ockl_sdot4(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+uint amd_mixed_dot(uchar4 a, uchar4 b, uint c, bool saturate) {
+  return __ockl_udot4(get_native_vector(a), get_native_vector(b), c, saturate);
+}
+__DEVICE__
+inline
+int amd_mixed_dot(int a, int b, int c, bool saturate) {
+    return __ockl_sdot8(a, b, c, saturate);
+}
+__DEVICE__
+inline
+uint amd_mixed_dot(uint a, uint b, uint c, bool saturate) {
+    return __ockl_udot8(a, b, c, saturate);
+}
+#endif
+
+#pragma pop_macro("__DEVICE__")
+#pragma pop_macro("__RETURN_TYPE")
+// For backward compatibility.
+// There are HIP applications e.g. TensorFlow, expecting __HIP_ARCH_* macros
+// defined after including math_functions.h.
+#if !defined(__HIPCC_RTC__)
+#include <hip/amd_detail/amd_hip_runtime.h>
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_surface_functions.h

@@ -0,0 +1,397 @@
+/*
+Copyright (c) 2018 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H
+
+#if defined(__cplusplus)
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/surface_types.h>
+#include <hip/hip_vector_types.h>
+#include <hip/amd_detail/texture_fetch_functions.h>
+#include <hip/amd_detail/ockl_image.h>
+#endif
+
+#if defined(__HIPCC_RTC__)
+#define __HOST_DEVICE__ __device__
+#else
+#define __HOST_DEVICE__ __host__ __device__
+#endif
+
+#define __HIP_SURFACE_OBJECT_PARAMETERS_INIT                                                            \
+    unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)surfObj;
+
+/**
+ *  @defgroup SurfaceAPI Surface API
+ *  @{
+ */
+
+// CUDA is using byte address, need map to pixel address for HIP
+static __HOST_DEVICE__ __forceinline__ int __hipGetPixelAddr(int x, int format, int order) {
+    /*
+    * use below format index to generate format LUT
+      typedef enum {
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT8 = 0,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT16 = 1,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT8 = 2,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT16 = 3,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT24 = 4,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555 = 5,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565 = 6,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_101010 = 7,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT8 = 8,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT16 = 9,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT32 = 10,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8 = 11,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16 = 12,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32 = 13,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_HALF_FLOAT = 14,
+        HSA_EXT_IMAGE_CHANNEL_TYPE_FLOAT = 15
+      } hsa_ext_image_channel_type_t;
+    */
+    static const int FormatLUT[] = { 0, 1, 0, 1, 3, 1, 1, 1, 0, 1, 2, 0, 1, 2, 1, 2 };
+    x = FormatLUT[format] == 3 ? x / FormatLUT[format] : x >> FormatLUT[format];
+
+    /*
+    * use below order index to generate order LUT
+      typedef enum {
+        HSA_EXT_IMAGE_CHANNEL_ORDER_A = 0,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_R = 1,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RX = 2,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RG = 3,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGX = 4,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RA = 5,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGB = 6,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGBX = 7,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_RGBA = 8,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_BGRA = 9,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_ARGB = 10,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_ABGR = 11,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SRGB = 12,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SRGBX = 13,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SRGBA = 14,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_SBGRA = 15,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_INTENSITY = 16,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_LUMINANCE = 17,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_DEPTH = 18,
+        HSA_EXT_IMAGE_CHANNEL_ORDER_DEPTH_STENCIL = 19
+      } hsa_ext_image_channel_order_t;
+    */
+    static const int OrderLUT[] = { 0, 0, 1, 1, 3, 1, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 0, 0, 0, 0 };
+    return x = OrderLUT[order] == 3 ? x / OrderLUT[order] : x >> OrderLUT[order];
+}
+
+/** \brief Reads the value at coordinate x from the one-dimensional surface.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The coordinate where the value will be read out.
+ *  \param boundaryMode [in] The boundary mode is currently ignored.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1Dread(T* data, hipSurfaceObject_t surfObj, int x,
+        int boundaryMode = hipBoundaryModeZero) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __ockl_image_load_1D(i, x);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the one-dimensional surface at coordinate x.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The coordinate where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1Dwrite(T data, hipSurfaceObject_t surfObj, int x) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_1D(i, x, tmp);
+}
+
+
+/** \brief Reads the value from the two-dimensional surface at coordinate x, y.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2Dread(T* data, hipSurfaceObject_t surfObj, int x, int y) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_2D(i, get_native_vector(coords));
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the two-dimensional surface at coordinate 
+ *         x, y.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2Dwrite(T data, hipSurfaceObject_t surfObj, int x, int y) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_2D(i, get_native_vector(coords), tmp);
+}
+
+/** \brief Reads the value from the three-dimensional surface at coordinate 
+ *         x, y, z.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param z [in] The z coordinate where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf3Dread(T* data, hipSurfaceObject_t surfObj, int x, int y, int z) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_3D(i), __ockl_image_channel_order_3D(i));
+    int4 coords{x, y, z, 0};
+    auto tmp = __ockl_image_load_3D(i, get_native_vector(coords));
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the three-dimensional surface at coordinate
+ *         x, y, z.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param z [in] The z coordinate where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf3Dwrite(T data, hipSurfaceObject_t surfObj, int x, int y, int z) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_3D(i), __ockl_image_channel_order_3D(i));
+    int4 coords{x, y, z, 0};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_3D(i, get_native_vector(coords), tmp);
+}
+
+/** \brief Reads the value from the one-dimensional layered surface at
+ *         coordinate x and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The coordinate where the value will be read out.
+ *  \param layer [in] The layer index where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1DLayeredread(T* data, hipSurfaceObject_t surfObj, int x, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __ockl_image_load_lod_1D(i, x, layer);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the one-dimensional layered surface at 
+ *         coordinate x and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf1DLayeredwrite(T data, hipSurfaceObject_t surfObj, int x, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_1D(i), __ockl_image_channel_order_1D(i));
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_lod_1D(i, x, layer, tmp);
+}
+
+/** \brief Reads the value from the two-dimensional layered surface at 
+ *         coordinate x, y and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param layer [in] The layer index where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2DLayeredread(T* data, hipSurfaceObject_t surfObj, int x, int y, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_lod_2D(i, get_native_vector(coords), layer);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the two-dimensional layered surface at
+ *         coordinate x, y and layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surf2DLayeredwrite(T data, hipSurfaceObject_t surfObj, int x, int y, int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_lod_2D(i, get_native_vector(coords), layer, tmp);
+}
+
+/** \brief Reads the value from the cubemap surface at coordinate x, y and
+ *         face index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param face [in] The face index where the value will be read out.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapread(T* data, hipSurfaceObject_t surfObj, int x, int y, int face) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_CM(i, get_native_vector(coords), face);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the cubemap surface at coordinate x, y and
+ *         face index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value is written to surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param face [in] The face index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapwrite(T data, hipSurfaceObject_t surfObj, int x, int y, int face) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_CM(i, get_native_vector(coords), face, tmp);
+}
+
+/** \brief Reads the value from the layered cubemap surface at coordinate x, y
+ *         and face, layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [out] The T type result is stored in this pointer.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the value will be read out.
+ *  \param y [in] The y coordinate where the value will be read out.
+ *  \param face [in] The face index where the value will be read out.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapLayeredread(T* data, hipSurfaceObject_t surfObj, int x, int y, int face,
+        int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __ockl_image_load_lod_CM(i, get_native_vector(coords), face, layer);
+    *data = __hipMapFrom<T>(tmp);
+}
+
+/** \brief Writes the value data to the layered cubemap surface at coordinate
+ *         x, y and face, layer index.
+ *
+ *  \tparam T The data type of the surface.
+ *  \param data [in] The T type value to write to the surface.
+ *  \param surfObj [in] The surface descriptor.
+ *  \param x [in] The x coordinate where the data will be written.
+ *  \param y [in] The y coordinate where the data will be written.
+ *  \param face [in] The face index where the data will be written.
+ *  \param layer [in] The layer index where the data will be written.
+ */
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void surfCubemapLayeredwrite(T* data, hipSurfaceObject_t surfObj, int x, int y, int face,
+        int layer) {
+    __HIP_SURFACE_OBJECT_PARAMETERS_INIT
+    x = __hipGetPixelAddr(x, __ockl_image_channel_data_type_2D(i), __ockl_image_channel_order_2D(i));
+    int2 coords{x, y};
+    auto tmp = __hipMapTo<float4::Native_vec_>(data);
+    __ockl_image_store_lod_CM(i, get_native_vector(coords), face, layer, tmp);
+}
+
+// Doxygen end group SurfaceAPI
+/**
+* @}
+*/
+
+#endif
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_warp_functions.h

@@ -0,0 +1,552 @@
+/*
+Copyright (c) 2022 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_WARP_FUNCTIONS_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_WARP_FUNCTIONS_H
+
+#if !defined(__HIPCC_RTC__)
+#include "device_library_decls.h"  // ockl warp functions
+#endif // !defined(__HIPCC_RTC__)
+
+#if defined(__has_attribute) && __has_attribute(maybe_undef)
+#define MAYBE_UNDEF __attribute__((maybe_undef))
+#else
+#define MAYBE_UNDEF
+#endif
+
+__device__ static inline unsigned __hip_ds_bpermute(int index, unsigned src) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = src;
+    tmp.i = __builtin_amdgcn_ds_bpermute(index, tmp.i);
+    return tmp.u;
+}
+
+__device__ static inline float __hip_ds_bpermutef(int index, float src) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = src;
+    tmp.i = __builtin_amdgcn_ds_bpermute(index, tmp.i);
+    return tmp.f;
+}
+
+__device__ static inline unsigned __hip_ds_permute(int index, unsigned src) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = src;
+    tmp.i = __builtin_amdgcn_ds_permute(index, tmp.i);
+    return tmp.u;
+}
+
+__device__ static inline float __hip_ds_permutef(int index, float src) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = src;
+    tmp.i = __builtin_amdgcn_ds_permute(index, tmp.i);
+    return tmp.f;
+}
+
+#define __hip_ds_swizzle(src, pattern)  __hip_ds_swizzle_N<(pattern)>((src))
+#define __hip_ds_swizzlef(src, pattern) __hip_ds_swizzlef_N<(pattern)>((src))
+
+template <int pattern>
+__device__ static inline unsigned __hip_ds_swizzle_N(unsigned int src) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = src;
+    tmp.i = __builtin_amdgcn_ds_swizzle(tmp.i, pattern);
+    return tmp.u;
+}
+
+template <int pattern>
+__device__ static inline float __hip_ds_swizzlef_N(float src) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = src;
+    tmp.i = __builtin_amdgcn_ds_swizzle(tmp.i, pattern);
+    return tmp.f;
+}
+
+#define __hip_move_dpp(src, dpp_ctrl, row_mask, bank_mask, bound_ctrl) \
+  __hip_move_dpp_N<(dpp_ctrl), (row_mask), (bank_mask), (bound_ctrl)>((src))
+
+template <int dpp_ctrl, int row_mask, int bank_mask, bool bound_ctrl>
+__device__ static inline int __hip_move_dpp_N(int src) {
+    return __builtin_amdgcn_mov_dpp(src, dpp_ctrl, row_mask, bank_mask,
+                                    bound_ctrl);
+}
+
+inline __device__ const struct final {
+  __device__
+  __attribute__((always_inline, const))
+    operator int() const noexcept {
+      return __builtin_amdgcn_wavefrontsize();
+    }
+} warpSize{};
+
+// warp vote function __all __any __ballot
+__device__
+inline
+int __all(int predicate) {
+    return __ockl_wfall_i32(predicate);
+}
+
+__device__
+inline
+int __any(int predicate) {
+    return __ockl_wfany_i32(predicate);
+}
+
+__device__
+inline
+unsigned long long int __ballot(int predicate) {
+    return __builtin_amdgcn_ballot_w64(predicate);
+}
+
+__device__
+inline
+unsigned long long int __ballot64(int predicate) {
+    return __ballot(predicate);
+}
+
+// See amd_warp_sync_functions.h for an explanation of this preprocessor flag.
+#if !defined(HIP_DISABLE_WARP_SYNC_BUILTINS)
+// Since threads in a wave do not make independent progress, __activemask()
+// always returns the exact active mask, i.e, all active threads in the wave.
+__device__
+inline
+unsigned long long __activemask() {
+  return __ballot(true);
+}
+#endif // HIP_DISABLE_WARP_SYNC_BUILTINS
+
+__device__ static inline unsigned int __lane_id() {
+    if (static_cast<int>(warpSize) == 32) return __builtin_amdgcn_mbcnt_lo(-1, 0);
+    return  __builtin_amdgcn_mbcnt_hi(
+        -1, __builtin_amdgcn_mbcnt_lo(-1, 0));
+}
+
+__device__
+inline
+int __shfl(MAYBE_UNDEF int var, int src_lane, int width = warpSize) {
+    int self = __lane_id();
+    int index = (src_lane & (width - 1)) + (self & ~(width-1));
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl(MAYBE_UNDEF unsigned int var, int src_lane, int width = warpSize) {
+     union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl(tmp.i, src_lane, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl(MAYBE_UNDEF float var, int src_lane, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl(tmp.i, src_lane, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl(MAYBE_UNDEF double var, int src_lane, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl(MAYBE_UNDEF long var, int src_lane, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl(static_cast<int>(var), src_lane, width));
+    #endif
+}
+__device__
+inline
+unsigned long __shfl(MAYBE_UNDEF unsigned long var, int src_lane, int width = warpSize) {
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl(static_cast<unsigned int>(var), src_lane, width));
+    #endif
+}
+__device__
+inline
+long long __shfl(MAYBE_UNDEF long long var, int src_lane, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+unsigned long long __shfl(MAYBE_UNDEF unsigned long long var, int src_lane, int width = warpSize) {
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl(tmp[0], src_lane, width);
+    tmp[1] = __shfl(tmp[1], src_lane, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+int __shfl_up(MAYBE_UNDEF int var, unsigned int lane_delta, int width = warpSize) {
+    int self = __lane_id();
+    int index = self - lane_delta;
+    index = (index < (self & ~(width-1)))?self:index;
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl_up(MAYBE_UNDEF unsigned int var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl_up(tmp.i, lane_delta, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl_up(MAYBE_UNDEF float var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl_up(tmp.i, lane_delta, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl_up(MAYBE_UNDEF double var, unsigned int lane_delta, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl_up(MAYBE_UNDEF long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl_up(static_cast<int>(var), lane_delta, width));
+    #endif
+}
+
+__device__
+inline
+unsigned long __shfl_up(MAYBE_UNDEF unsigned long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl_up(static_cast<unsigned int>(var), lane_delta, width));
+    #endif
+}
+
+__device__
+inline
+long long __shfl_up(MAYBE_UNDEF long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+unsigned long long __shfl_up(MAYBE_UNDEF unsigned long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_up(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_up(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+int __shfl_down(MAYBE_UNDEF int var, unsigned int lane_delta, int width = warpSize) {
+    int self = __lane_id();
+    int index = self + lane_delta;
+    index = (int)((self&(width-1))+lane_delta) >= width?self:index;
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl_down(MAYBE_UNDEF unsigned int var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl_down(tmp.i, lane_delta, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl_down(MAYBE_UNDEF float var, unsigned int lane_delta, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl_down(tmp.i, lane_delta, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl_down(MAYBE_UNDEF double var, unsigned int lane_delta, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl_down(MAYBE_UNDEF long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl_down(static_cast<int>(var), lane_delta, width));
+    #endif
+}
+__device__
+inline
+unsigned long __shfl_down(MAYBE_UNDEF unsigned long var, unsigned int lane_delta, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl_down(static_cast<unsigned int>(var), lane_delta, width));
+    #endif
+}
+__device__
+inline
+long long __shfl_down(MAYBE_UNDEF long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+unsigned long long __shfl_down(MAYBE_UNDEF unsigned long long var, unsigned int lane_delta, int width = warpSize)
+{
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_down(tmp[0], lane_delta, width);
+    tmp[1] = __shfl_down(tmp[1], lane_delta, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+__device__
+inline
+int __shfl_xor(MAYBE_UNDEF int var, int lane_mask, int width = warpSize) {
+    int self = __lane_id();
+    int index = self^lane_mask;
+    index = index >= ((self+width)&~(width-1))?self:index;
+    return __builtin_amdgcn_ds_bpermute(index<<2, var);
+}
+__device__
+inline
+unsigned int __shfl_xor(MAYBE_UNDEF unsigned int var, int lane_mask, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.u = var;
+    tmp.i = __shfl_xor(tmp.i, lane_mask, width);
+    return tmp.u;
+}
+__device__
+inline
+float __shfl_xor(MAYBE_UNDEF float var, int lane_mask, int width = warpSize) {
+    union { int i; unsigned u; float f; } tmp; tmp.f = var;
+    tmp.i = __shfl_xor(tmp.i, lane_mask, width);
+    return tmp.f;
+}
+__device__
+inline
+double __shfl_xor(MAYBE_UNDEF double var, int lane_mask, int width = warpSize) {
+    static_assert(sizeof(double) == 2 * sizeof(int), "");
+    static_assert(sizeof(double) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    double tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+long __shfl_xor(MAYBE_UNDEF long var, int lane_mask, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long) == sizeof(__hip_uint64_t), "");
+
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(long) == sizeof(int), "");
+    return static_cast<long>(__shfl_xor(static_cast<int>(var), lane_mask, width));
+    #endif
+}
+__device__
+inline
+unsigned long __shfl_xor(MAYBE_UNDEF unsigned long var, int lane_mask, int width = warpSize)
+{
+    #ifndef _MSC_VER
+    static_assert(sizeof(unsigned long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long) == sizeof(__hip_uint64_t), "");
+
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+    #else
+    static_assert(sizeof(unsigned long) == sizeof(unsigned int), "");
+    return static_cast<unsigned long>(__shfl_xor(static_cast<unsigned int>(var), lane_mask, width));
+    #endif
+}
+__device__
+inline
+long long __shfl_xor(MAYBE_UNDEF long long var, int lane_mask, int width = warpSize)
+{
+    static_assert(sizeof(long long) == 2 * sizeof(int), "");
+    static_assert(sizeof(long long) == sizeof(__hip_uint64_t), "");
+    int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+__device__
+inline
+unsigned long long __shfl_xor(MAYBE_UNDEF unsigned long long var, int lane_mask, int width = warpSize)
+{
+    static_assert(sizeof(unsigned long long) == 2 * sizeof(unsigned int), "");
+    static_assert(sizeof(unsigned long long) == sizeof(__hip_uint64_t), "");
+    unsigned int tmp[2]; __builtin_memcpy(tmp, &var, sizeof(tmp));
+    tmp[0] = __shfl_xor(tmp[0], lane_mask, width);
+    tmp[1] = __shfl_xor(tmp[1], lane_mask, width);
+    __hip_uint64_t tmp0 = (static_cast<__hip_uint64_t>(tmp[1]) << 32ull) | static_cast<__hip_uint32_t>(tmp[0]);
+    unsigned long long tmp1;  __builtin_memcpy(&tmp1, &tmp0, sizeof(tmp0));
+    return tmp1;
+}
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/amd_warp_sync_functions.h

@@ -0,0 +1,746 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+// Warp sync builtins (with explicit mask argument) introduced in ROCm 6.2 as a
+// preview to allow end-users to adapt to the new interface involving 64-bit
+// masks. These are enabled by default, and can be disabled by setting the macro
+// "HIP_DISABLE_WARP_SYNC_BUILTINS". This arrangement also applies to the
+// __activemask() builtin defined in amd_warp_functions.h.
+#if !defined(HIP_DISABLE_WARP_SYNC_BUILTINS)
+
+#if !defined(__HIPCC_RTC__)
+#include "amd_warp_functions.h"
+#include "amd_device_functions.h"
+#include "hip_assert.h"
+#include <functional>
+#include <algorithm>
+#endif
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_add_i32(int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_add_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_min_i32(int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_min_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_max_i32(int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_max_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_and_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_or_u32(unsigned int);
+extern "C" __device__ __attribute__((const)) unsigned int __ockl_wfred_xor_u32(unsigned int);
+
+#ifdef HIP_ENABLE_EXTRA_WARP_SYNC_TYPES
+// this macro enable types that are not in CUDA
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_add_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_add_u64(unsigned long long);
+extern "C" __device__ __attribute__((const)) float __ockl_wfred_add_f32(float);
+extern "C" __device__ __attribute__((const)) double __ockl_wfred_add_f64(double);
+
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_min_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_min_u64(unsigned long long);
+extern "C" __device__ __attribute__((const)) float __ockl_wfred_min_f32(float);
+extern "C" __device__ __attribute__((const)) double __ockl_wfred_min_f64(double);
+
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_max_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_max_u64(unsigned long long);
+extern "C" __device__ __attribute__((const)) float __ockl_wfred_max_f32(float);
+extern "C" __device__ __attribute__((const)) double __ockl_wfred_max_f64(double);
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_and_i32(int);
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_and_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_and_u64(unsigned long long);
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_or_i32(int);
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_or_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_or_u64(unsigned long long);
+
+extern "C" __device__ __attribute__((const)) int __ockl_wfred_xor_i32(int);
+extern "C" __device__ __attribute__((const)) long long __ockl_wfred_xor_i64(long long);
+extern "C" __device__ __attribute__((const)) unsigned long long __ockl_wfred_xor_u64(unsigned long long);
+
+#endif
+
+template <typename T>
+__device__ inline
+T __hip_readfirstlane(T val) {
+  // In theory, behaviour is undefined when reading from a union member other
+  // than the member that was last assigned to, but it works in practice because
+  // we rely on the compiler to do the reasonable thing.
+  union {
+    unsigned long long l;
+    T d;
+  } u;
+  u.d = val;
+  // NOTE: The builtin returns int, so we first cast it to unsigned int and only
+  // then extend it to 64 bits.
+  unsigned long long lower = (unsigned)__builtin_amdgcn_readfirstlane(u.l);
+  unsigned long long upper =
+      (unsigned)__builtin_amdgcn_readfirstlane(u.l >> 32);
+  u.l = (upper << 32) | lower;
+  return u.d;
+}
+
+// When compiling for wave32 mode, ignore the upper half of the 64-bit mask.
+#define __hip_adjust_mask_for_wave32(MASK)            \
+  do {                                          \
+    if (static_cast<int>(warpSize) == 32) MASK &= 0xFFFFFFFF;     \
+  } while (0)
+
+// We use a macro to expand each builtin into a waterfall that implements the
+// mask semantics:
+//
+// 1. The mask argument may be divergent.
+// 2. Each active thread must have its own bit set in its own mask value.
+// 3. For a given mask value, all threads that are mentioned in the mask must
+//    execute the same static instance of the builtin with the same mask.
+// 4. The union of all mask values supplied at a static instance must be equal
+//    to the activemask at the program point.
+//
+// Thus, the mask argument partitions the set of currently active threads in the
+// wave into disjoint subsets that cover all active threads.
+//
+// Implementation notes:
+// ---------------------
+//
+// We implement this as a waterfall loop that executes the builtin for each
+// subset separately. The return value is a divergent value across the active
+// threads. The value for inactive threads is defined by each builtin
+// separately.
+//
+// As long as every mask value is non-zero, we don't need to check if a lane
+// specifies itself in the mask; that is done by the later assertion where all
+// chosen lanes must be in the chosen mask.
+
+#define __hip_check_mask(MASK)                                                 \
+  do {                                                                         \
+    __hip_assert(MASK && "mask must be non-zero");                             \
+    bool done = false;                                                         \
+    while (__any(!done)) {                                                     \
+      if (!done) {                                                             \
+        auto chosen_mask = __hip_readfirstlane(MASK);                          \
+        if (MASK == chosen_mask) {                                             \
+          __hip_assert(MASK == __ballot(true) &&                               \
+                       "all threads specified in the mask"                     \
+                       " must execute the same operation with the same mask"); \
+          done = true;                                                         \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+  } while(0)
+
+#define __hip_do_sync(RETVAL, FUNC, MASK, ...)                                 \
+  do {                                                                         \
+    __hip_assert(MASK && "mask must be non-zero");                             \
+    bool done = false;                                                         \
+    while (__any(!done)) {                                                     \
+      if (!done) {                                                             \
+        auto chosen_mask = __hip_readfirstlane(MASK);                          \
+        if (MASK == chosen_mask) {                                             \
+          __hip_assert(MASK == __ballot(true) &&                               \
+                       "all threads specified in the mask"                     \
+                       " must execute the same operation with the same mask"); \
+          RETVAL = FUNC(__VA_ARGS__);                                          \
+          done = true;                                                         \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+  } while(0)
+
+__device__ inline void __syncwarp() {
+  __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
+  __builtin_amdgcn_wave_barrier();
+  __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "wavefront");
+}
+
+template <typename MaskT> __device__ inline void __syncwarp(MaskT mask) {
+  static_assert(__hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+                "The mask must be a 64-bit integer. "
+                "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_check_mask(mask);
+  return __syncwarp();
+}
+
+// __all_sync, __any_sync, __ballot_sync
+
+template <typename MaskT>
+__device__ inline
+unsigned long long __ballot_sync(MaskT mask, int predicate) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __ballot(predicate) & mask;
+}
+
+template <typename MaskT>
+__device__ inline
+int __all_sync(MaskT mask, int predicate) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  return __ballot_sync(mask, predicate) == mask;
+}
+
+template <typename MaskT>
+__device__ inline
+int __any_sync(MaskT mask, int predicate) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  return __ballot_sync(mask, predicate) != 0;
+}
+
+// __match_any, __match_all and sync variants
+
+template <typename T>
+__device__ inline
+unsigned long long __match_any(T value) {
+  static_assert(
+      (__hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value) &&
+          (sizeof(T) == 4 || sizeof(T) == 8),
+      "T can be int, unsigned int, long, unsigned long, long long, unsigned "
+      "long long, float or double.");
+  bool done = false;
+  unsigned long long retval = 0;
+
+  while (__any(!done)) {
+    if (!done) {
+      T chosen = __hip_readfirstlane(value);
+      if (chosen == value) {
+        retval = __activemask();
+        done = true;
+      }
+    }
+  }
+
+  return retval;
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+unsigned long long __match_any_sync(MaskT mask, T value) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __match_any(value) & mask;
+}
+
+template <typename T>
+__device__ inline
+unsigned long long __match_all(T value, int* pred) {
+  static_assert(
+      (__hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value) &&
+          (sizeof(T) == 4 || sizeof(T) == 8),
+      "T can be int, unsigned int, long, unsigned long, long long, unsigned "
+      "long long, float or double.");
+  T first = __hip_readfirstlane(value);
+  if (__all(first == value)) {
+    *pred = true;
+    return __activemask();
+  } else {
+    *pred = false;
+    return 0;
+  }
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+unsigned long long __match_all_sync(MaskT mask, T value, int* pred) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  MaskT retval = 0;
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_do_sync(retval, __match_all, mask, value, pred);
+  return retval;
+}
+
+// various variants of shfl
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_sync(MaskT mask, T var, int srcLane,
+              int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl(var, srcLane, width);
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_up_sync(MaskT mask, T var, unsigned int delta,
+                                   int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl_up(var, delta, width);
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_down_sync(MaskT mask, T var, unsigned int delta,
+                                     int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl_down(var, delta, width);
+}
+
+template <typename MaskT, typename T>
+__device__ inline
+T __shfl_xor_sync(MaskT mask, T var, int laneMask,
+                                    int width = warpSize) {
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  __hip_check_mask(mask);
+  return __shfl_xor(var, laneMask, width);
+}
+
+template <typename MaskT, typename T, typename BinaryOp, typename WfReduce>
+__device__ inline T __reduce_op_sync(MaskT mask, T val, BinaryOp op, WfReduce wfReduce)
+{
+  using permuteType =
+    typename __hip_internal::conditional<sizeof(T) == 4 || sizeof(T) == 2, T, unsigned int>::type;
+  static constexpr auto kMaskNumBits = sizeof(MaskT) * 8;
+  static_assert(
+      __hip_internal::is_integral<MaskT>::value && sizeof(MaskT) == 8,
+      "The mask must be a 64-bit integer. "
+      "Implicitly promoting a smaller integer is almost always an error.");
+  __hip_adjust_mask_for_wave32(mask);
+  unsigned int laneId;
+  unsigned int maskIdx;
+  // next bit to aggregate with
+  int nextBit;
+
+  // if doing the binary reduction tree, this will increase by two in every iteration
+  int modulo = 1;
+  int leadingZeroes = __clzll(mask);
+  int firstLane;
+  int lastLane = kMaskNumBits - leadingZeroes - 1;
+  int maskNumBits;
+  int numIterations;
+  // unsigned int[2] is used when T is 64-bit wide
+  typename __hip_internal::conditional<sizeof(T) == 4 || sizeof(T) == 2, permuteType, permuteType[2]>::type result, permuteResult;
+  auto backwardPermute = [](int index, permuteType val) {
+    if constexpr (__hip_internal::is_integral<T>::value || __hip_internal::is_same<T, double>::value)
+      return __hip_ds_bpermute(index, val);
+    else
+      return __hip_ds_bpermutef(index, val);
+  };
+
+  __hip_check_mask(mask);
+  maskNumBits = __popcll(mask);
+
+#ifdef __OPTIMIZE__ // at the time of this writing the ockl wfred functions do not compile when using -O0
+  if (maskNumBits == lastLane + 1)
+    // this means the mask "does not have holes", and starts from 0; we can use a specific intrinsic
+    // to calculate the aggregated result
+    return wfReduce(val);
+#endif
+
+  firstLane = __builtin_ctzll(mask);
+  laneId = __ockl_lane_u32();
+  nextBit = laneId;
+  // the number of iterations needs to be at least log2(number of bits on)
+  numIterations = sizeof(int) * 8 - __clz(maskNumBits);
+
+  if (!(maskNumBits & (maskNumBits - 1)))
+    // the number of bits in the mask is a power of 2
+    numIterations -= 1;
+
+  maskIdx = __popcll(((1ul << laneId) - 1) & mask);
+  mask >>= laneId;
+  mask >>= 1ul;
+
+  if constexpr(sizeof(T) == 4 || sizeof(T) == 2)
+    result = val;
+  else
+    __builtin_memcpy(&result, &val, sizeof(T));
+
+  // add the values from the lanes using a reduction tree (first the threads with even-numbered
+  // lanes, then multiples of 4, then 8, ...
+  while (numIterations) {
+    int offset = modulo >> 1;
+    int increment = modulo - offset;
+    int nextPos = maskIdx + offset + increment;
+    bool insideLanes = nextPos < maskNumBits;
+
+    if (insideLanes) {
+      int next;
+
+      // find the position to aggregate with; although we could just call fns64() that will probably
+      // be very slow when called multiple times in this for loop; this is equivalent
+      for (int i = 0; i < increment; i++) {
+        next = __builtin_ctzll(mask) + 1;
+        mask >>= next;
+        nextBit += next;
+      }
+    }
+
+    if constexpr (sizeof(T) == 2) {
+      union { int i; T f; } tmp;
+
+      tmp.f = result;
+      tmp.i = __hip_ds_bpermute(nextBit << 2, tmp.i);
+      permuteResult = tmp.f;
+    } else if constexpr (sizeof(T) == 4)
+      permuteResult = backwardPermute(nextBit << 2, result);
+    else {
+      // ds_bpermute only deals with 32-bit sizes, so for 64-bit types
+      // we need to call the permute twice for each half
+      permuteResult[0] = backwardPermute(nextBit << 2, result[0]);
+      permuteResult[1] = backwardPermute(nextBit << 2, result[1]);
+    }
+
+    if (insideLanes) {
+      if constexpr (sizeof(T) == 4 || sizeof(T) == 2)
+        result = op(result, permuteResult);
+      else {
+        T tmp;
+        unsigned long long rhs = (static_cast<unsigned long long>(permuteResult[1]) << 32) | permuteResult[0];
+
+        __builtin_memcpy(&tmp, &result, sizeof(T));
+        tmp = op(tmp, *reinterpret_cast<T*>(&rhs));
+        __builtin_memcpy(&result, &tmp, sizeof(T));
+      }
+    }
+
+    modulo <<= 1;
+    numIterations--;
+  }
+
+  if constexpr (sizeof(T) == 2) {
+    union { int i; T f; } tmp;
+    tmp.f = result;
+    tmp.i = __hip_ds_bpermute(firstLane << 2, tmp.i);
+    return tmp.f;
+  } else if constexpr (sizeof(T) == 4)
+    return backwardPermute(firstLane << 2, result);
+  else {
+    auto tmp = (static_cast<unsigned long long>(backwardPermute(firstLane << 2, result[1])) << 32) |
+                static_cast<unsigned int>(backwardPermute(firstLane << 2, result[0]));
+    return *reinterpret_cast<T*>(&tmp);
+  }
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_add_sync(MaskT mask, int val)
+{
+  // although C++ has std::plus and other functors, we do not use them because
+  // they are in the header <functional> and they were causing problem with hipRTC
+  // at this time
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_add_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_min_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_min_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_max_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_max_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_or_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_and_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned int __reduce_xor_sync(MaskT mask, unsigned int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs) == 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_u32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+#ifdef HIP_ENABLE_EXTRA_WARP_SYNC_TYPES
+template <typename MaskT>
+__device__ inline long long __reduce_add_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_add_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline float __reduce_add_sync(MaskT mask, float val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_f32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline double __reduce_add_sync(MaskT mask, double val)
+{
+  auto op = [](decltype(val)& a, decltype(val)& b) { return a + b; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_add_f64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_min_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_min_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline float __reduce_min_sync(MaskT mask, float val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_f32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline double __reduce_min_sync(MaskT mask, double val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return rhs < lhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_min_f64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_max_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_max_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline float __reduce_max_sync(MaskT mask, float val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_f32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline double __reduce_max_sync(MaskT mask, double val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs < rhs? rhs : lhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_max_f64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_and_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_and_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_and_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs && rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_and_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_or_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_or_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_or_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return lhs || rhs; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_or_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline int __reduce_xor_sync(MaskT mask, int val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs) == 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_i32(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline long long __reduce_xor_sync(MaskT mask, long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs) == 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_i64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+template <typename MaskT>
+__device__ inline unsigned long long __reduce_xor_sync(MaskT mask, unsigned long long val)
+{
+  auto op = [](decltype(val) lhs, decltype(val) rhs) { return (!lhs) != (!rhs)== 1; };
+  auto wfReduce = [](decltype(val) v) { return __ockl_wfred_xor_u64(v); };
+
+  return __reduce_op_sync(mask, val, op, wfReduce);
+}
+
+#undef __hip_do_sync
+#undef __hip_check_mask
+#undef __hip_adjust_mask_for_wave32
+
+#endif  // HIP_ENABLE_EXTRA_WARP_SYNC_TYPES
+#endif // HIP_DISABLE_WARP_SYNC_BUILTINS

projects/clr/hipamd/include/hip/amd_detail/concepts.hpp

@@ -0,0 +1,30 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+namespace hip_impl  // Documentation only.
+{
+#define requires(...)
+
+#define FunctionalProcedure typename
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/device_library_decls.h

@@ -0,0 +1,133 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/device_library_decls.h
+ *  @brief Contains declarations for types and functions in device library.
+ *         Uses __hip_int64_t and __hip_uint64_t instead of long, long long, unsigned
+ *         long and unsigned long long types for device library API
+ *         declarations.
+ */
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
+
+#if !defined(__HIPCC_RTC__)
+#include "hip/amd_detail/host_defines.h"
+#if __cplusplus
+#include <cstdint>
+#else
+#include <stdint.h>
+#endif
+#endif
+
+typedef unsigned char uchar;
+typedef unsigned short ushort;
+typedef unsigned int uint;
+typedef unsigned long ulong;
+typedef unsigned long long ullong;
+
+extern "C" __device__ __attribute__((const)) bool __ockl_wfany_i32(int);
+extern "C" __device__ __attribute__((const)) bool __ockl_wfall_i32(int);
+extern "C" __device__ uint __ockl_activelane_u32(void);
+
+extern "C" __device__ __attribute__((const)) uint __ockl_mul24_u32(uint, uint);
+extern "C" __device__ __attribute__((const)) int __ockl_mul24_i32(int, int);
+extern "C" __device__ __attribute__((const)) uint __ockl_mul_hi_u32(uint, uint);
+extern "C" __device__ __attribute__((const)) int __ockl_mul_hi_i32(int, int);
+extern "C" __device__ __attribute__((const)) uint __ockl_sadd_u32(uint, uint, uint);
+
+extern "C" __device__ __attribute__((const)) uchar __ockl_clz_u8(uchar);
+extern "C" __device__ __attribute__((const)) ushort __ockl_clz_u16(ushort);
+extern "C" __device__ __attribute__((const)) uint __ockl_clz_u32(uint);
+extern "C" __device__ __attribute__((const)) __hip_uint64_t __ockl_clz_u64(__hip_uint64_t);
+
+extern "C" __device__ __attribute__((const)) float __ocml_fmin_f32(float, float);
+extern "C" __device__ __attribute__((const)) float __ocml_fmax_f32(float, float);
+
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_f64(double);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_f64(double);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_f64(double);
+
+extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
+extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
+extern "C" __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
+
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_s32(int);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_s32(int);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_s32(int);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_u32(__hip_uint32_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_u32(__hip_uint32_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_u32(__hip_uint32_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtn_f32_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtp_f32_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) float __ocml_cvtrtz_f32_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtn_f64_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtp_f64_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtz_f64_s64(__hip_int64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtn_f64_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtp_f64_u64(__hip_uint64_t);
+extern "C" __device__ __attribute__((const)) double __ocml_cvtrtz_f64_u64(__hip_uint64_t);
+
+extern "C" __device__ __attribute__((convergent)) void __ockl_gws_init(uint nwm1, uint rid);
+extern "C" __device__ __attribute__((convergent)) void __ockl_gws_barrier(uint nwm1, uint rid);
+
+extern "C" __device__ __attribute__((const)) __hip_uint32_t __ockl_lane_u32();
+extern "C" __device__ __attribute__((const)) int __ockl_grid_is_valid(void);
+extern "C" __device__ __attribute__((convergent)) void __ockl_grid_sync(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_num_grids(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_grid_rank(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_size(void);
+extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_thread_rank(void);
+extern "C" __device__ __attribute__((const)) int __ockl_multi_grid_is_valid(void);
+extern "C" __device__ __attribute__((convergent)) void __ockl_multi_grid_sync(void);
+
+extern "C" __device__ void __ockl_atomic_add_noret_f32(float*, float);
+
+extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_add_i32(int a);
+extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_and_i32(int a);
+extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_or_i32(int a);
+
+extern "C" __device__ __hip_uint64_t __ockl_fprintf_stderr_begin();
+extern "C" __device__ __hip_uint64_t __ockl_fprintf_append_args(__hip_uint64_t msg_desc, __hip_uint32_t num_args,
+                                                          __hip_uint64_t value0, __hip_uint64_t value1,
+                                                          __hip_uint64_t value2, __hip_uint64_t value3,
+                                                          __hip_uint64_t value4, __hip_uint64_t value5,
+                                                          __hip_uint64_t value6, __hip_uint32_t is_last);
+extern "C" __device__ __hip_uint64_t __ockl_fprintf_append_string_n(__hip_uint64_t msg_desc, const char* data,
+                                                              __hip_uint64_t length, __hip_uint32_t is_last);
+
+// Introduce local address space
+#define __local __attribute__((address_space(3)))
+
+#ifdef __HIP_DEVICE_COMPILE__
+__device__ inline static __local void* __to_local(unsigned x) { return (__local void*)x; }
+#endif //__HIP_DEVICE_COMPILE__
+
+// Using hip.amdgcn.bc - sync threads
+#define __CLK_LOCAL_MEM_FENCE    0x01
+typedef unsigned __cl_mem_fence_flags;
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/functional_grid_launch.hpp

@@ -0,0 +1,218 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "concepts.hpp"
+#include "helpers.hpp"
+#include "program_state.hpp"
+#include "hip_runtime_api.h"
+
+#include <cstdint>
+#include <cstring>
+#include <stdexcept>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices,
+                                               unsigned int flags, hip_impl::program_state& ps);
+
+hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim,
+                                    dim3 blockDim, void** args,
+                                    size_t sharedMem, hipStream_t stream,
+                                    hip_impl::program_state& ps);
+
+hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
+                                                 int  numDevices,
+                                                 unsigned int flags,
+                                                 hip_impl::program_state& ps);
+
+#pragma GCC visibility push(hidden)
+
+namespace hip_impl {
+template <typename T, typename std::enable_if<std::is_integral<T>{}>::type* = nullptr>
+inline T round_up_to_next_multiple_nonnegative(T x, T y) {
+    T tmp = x + y - 1;
+    return tmp - tmp % y;
+}
+
+template <
+    std::size_t n,
+    typename... Ts,
+    typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
+inline hip_impl::kernarg make_kernarg(
+    const std::tuple<Ts...>&,
+    const kernargs_size_align&,
+    hip_impl::kernarg kernarg) {
+    return kernarg;
+}
+
+template <
+    std::size_t n,
+    typename... Ts,
+    typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
+inline hip_impl::kernarg make_kernarg(
+    const std::tuple<Ts...>& formals,
+    const kernargs_size_align& size_align,
+    hip_impl::kernarg kernarg) {
+    using T = typename std::tuple_element<n, std::tuple<Ts...>>::type;
+
+    static_assert(
+        !std::is_reference<T>{},
+        "A __global__ function cannot have a reference as one of its "
+            "arguments.");
+    #if defined(HIP_STRICT)
+        static_assert(
+            std::is_trivially_copyable<T>{},
+            "Only TriviallyCopyable types can be arguments to a __global__ "
+                "function");
+    #endif
+
+    kernarg.resize(round_up_to_next_multiple_nonnegative(
+        kernarg.size(), size_align.alignment(n)) + size_align.size(n));
+
+    std::memcpy(
+        kernarg.data() + kernarg.size() - size_align.size(n),
+        &std::get<n>(formals),
+        size_align.size(n));
+    return make_kernarg<n + 1>(formals, size_align, std::move(kernarg));
+}
+
+template <typename... Formals, typename... Actuals>
+inline hip_impl::kernarg make_kernarg(
+    void (*kernel)(Formals...), std::tuple<Actuals...> actuals) {
+    static_assert(sizeof...(Formals) == sizeof...(Actuals),
+        "The count of formal arguments must match the count of actuals.");
+
+    if (sizeof...(Formals) == 0) return {};
+
+    std::tuple<Formals...> to_formals{std::move(actuals)};
+    hip_impl::kernarg kernarg;
+    kernarg.reserve(sizeof(to_formals));
+
+    auto& ps = hip_impl::get_program_state();
+    return make_kernarg<0>(to_formals,
+                           ps.get_kernargs_size_align(
+                               reinterpret_cast<std::uintptr_t>(kernel)),
+                           std::move(kernarg));
+}
+
+
+HIP_INTERNAL_EXPORTED_API hsa_agent_t target_agent(hipStream_t stream);
+
+inline
+__attribute__((visibility("hidden")))
+void hipLaunchKernelGGLImpl(
+    std::uintptr_t function_address,
+    const dim3& numBlocks,
+    const dim3& dimBlocks,
+    std::uint32_t sharedMemBytes,
+    hipStream_t stream,
+    void** kernarg) {
+
+    const auto& kd = hip_impl::get_program_state().kernel_descriptor(function_address,
+                                                               target_agent(stream));
+
+    hipModuleLaunchKernel(kd, numBlocks.x, numBlocks.y, numBlocks.z,
+                          dimBlocks.x, dimBlocks.y, dimBlocks.z, sharedMemBytes,
+                          stream, nullptr, kernarg);
+}
+} // Namespace hip_impl.
+
+
+template <class T>
+inline
+hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize,
+    T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0) {
+
+    using namespace hip_impl;
+
+    hip_impl::hip_init();
+    auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
+                                                   target_agent(0));
+
+    return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
+                                      dynSharedMemPerBlk, blockSizeLimit);
+}
+
+template <class T>
+inline
+hipError_t hipOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize,
+    T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0, unsigned int  flags = 0 ) {
+
+    using namespace hip_impl;
+
+    hip_impl::hip_init();
+    if(flags != hipOccupancyDefault) return hipErrorNotSupported;
+    auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
+                                                   target_agent(0));
+
+    return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
+                                      dynSharedMemPerBlk, blockSizeLimit);
+}
+
+template <typename... Args, typename F = void (*)(Args...)>
+inline
+void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
+                        std::uint32_t sharedMemBytes, hipStream_t stream,
+                        Args... args) {
+    hip_impl::hip_init();
+    auto kernarg = hip_impl::make_kernarg(kernel, std::tuple<Args...>{std::move(args)...});
+    std::size_t kernarg_size = kernarg.size();
+
+    void* config[]{
+        HIP_LAUNCH_PARAM_BUFFER_POINTER,
+        kernarg.data(),
+        HIP_LAUNCH_PARAM_BUFFER_SIZE,
+        &kernarg_size,
+        HIP_LAUNCH_PARAM_END};
+
+    hip_impl::hipLaunchKernelGGLImpl(reinterpret_cast<std::uintptr_t>(kernel),
+                                     numBlocks, dimBlocks, sharedMemBytes,
+                                     stream, &config[0]);
+}
+
+template <typename F>
+inline
+__attribute__((visibility("hidden")))
+hipError_t hipLaunchCooperativeKernel(F f, dim3 gridDim, dim3 blockDim,
+                                      void** args, size_t sharedMem,
+                                      hipStream_t stream) {
+    hip_impl::hip_init();
+    auto& ps = hip_impl::get_program_state();
+    return hipLaunchCooperativeKernel(reinterpret_cast<void*>(f), gridDim,
+                                      blockDim, args, sharedMem, stream, ps);
+}
+
+inline
+__attribute__((visibility("hidden")))
+hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
+                                                 int  numDevices,
+                                                 unsigned int  flags) {
+
+    hip_impl::hip_init();
+    auto& ps = hip_impl::get_program_state();
+    return hipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, ps);
+}
+
+#pragma GCC visibility pop

projects/clr/hipamd/include/hip/amd_detail/grid_launch.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <hc_defines.h>
+
+#define GRID_LAUNCH_VERSION 20
+
+// Extern definitions
+namespace hc{
+class completion_future;
+class accelerator_view;
+}
+
+
+// 3 dim structure for groups and grids.
+typedef struct gl_dim3
+{
+  int x,y,z;
+  gl_dim3(uint32_t _x=1, uint32_t _y=1, uint32_t _z=1) : x(_x), y(_y), z(_z) {};
+} gl_dim3;
+
+typedef enum gl_barrier_bit {
+    barrier_bit_queue_default,
+    barrier_bit_none,
+    barrier_bit_wait,
+} gl_barrier_bit;
+
+
+// grid_launch_parm contains information used to launch the kernel.
+typedef struct grid_launch_parm
+{
+  //! Grid dimensions
+  gl_dim3      grid_dim;
+
+  //! Group dimensions
+  gl_dim3      group_dim;
+
+  //! Amount of dynamic group memory to use with the kernel launch.
+  //! This memory is in addition to the amount used statically in the kernel.
+  unsigned int  dynamic_group_mem_bytes;
+
+  //! Control setting of barrier bit on per-packet basis:
+  //! See gl_barrier_bit description.  
+  //! Placeholder, is not used to control packet dispatch yet
+  enum gl_barrier_bit barrier_bit;
+
+  //! Value of packet fences to apply to launch.
+  //! The correspond to the value of bits 9:14 in the AQL packet,
+  //! see HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE and hsa_fence_scope_t.
+  unsigned int  launch_fence;
+
+  //! Pointer to the accelerator_view where the kernel should execute.
+  //! If NULL, the default view on the default accelerator is used.
+  hc::accelerator_view  *av;
+
+  //! Pointer to the completion_future used to track the status of the command.
+  //! If NULL, the command does not write status.  In this case, 
+  //! synchronization can be enforced with queue-level waits or 
+  //! waiting on younger commands.
+  hc::completion_future *cf;
+
+  grid_launch_parm() = default;
+} grid_launch_parm;
+
+
+extern void init_grid_launch(grid_launch_parm *gl);

projects/clr/hipamd/include/hip/amd_detail/grid_launch.hpp

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "grid_launch.h"
+#include "hc.hpp"
+
+class grid_launch_parm_cxx : public grid_launch_parm
+{
+public:
+  grid_launch_parm_cxx() = default;
+
+  // customized serialization: don't need av and cf in kernel
+  __attribute__((annotate("serialize")))
+  void __cxxamp_serialize(Kalmar::Serialize& s) const {
+    s.Append(sizeof(int), &grid_dim.x);
+    s.Append(sizeof(int), &grid_dim.y);
+    s.Append(sizeof(int), &grid_dim.z);
+    s.Append(sizeof(int), &group_dim.x);
+    s.Append(sizeof(int), &group_dim.y);
+    s.Append(sizeof(int), &group_dim.z);
+  }
+
+  __attribute__((annotate("user_deserialize")))
+  grid_launch_parm_cxx(int grid_dim_x,  int grid_dim_y,  int grid_dim_z,
+                   int group_dim_x, int group_dim_y, int group_dim_z) {
+    grid_dim.x  = grid_dim_x;
+    grid_dim.y  = grid_dim_y;
+    grid_dim.z  = grid_dim_z;
+    group_dim.x = group_dim_x;
+    group_dim.y = group_dim_y;
+    group_dim.z = group_dim_z;
+  }
+};
+
+
+extern inline void grid_launch_init(grid_launch_parm *lp) {
+  lp->grid_dim.x = lp->grid_dim.y = lp->grid_dim.z = 1;
+
+  lp->group_dim.x = lp->group_dim.y = lp->group_dim.z = 1;
+
+  lp->dynamic_group_mem_bytes = 0;
+
+  lp->barrier_bit = barrier_bit_queue_default;
+  lp->launch_fence = -1;
+
+  // TODO - set to NULL?
+  static hc::accelerator_view av = hc::accelerator().get_default_view();
+  lp->av = &av;
+  lp->cf = NULL;
+}
+

projects/clr/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp

@@ -0,0 +1,26 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#pragma once
+
+#if GENERIC_GRID_LAUNCH == 1
+#include "macro_based_grid_launch.hpp"
+#endif  // GENERIC_GRID_LAUNCH

projects/clr/hipamd/include/hip/amd_detail/helpers.hpp

@@ -0,0 +1,137 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "concepts.hpp"
+
+#include <type_traits>  // For std::conditional, std::decay, std::enable_if,
+                        // std::false_type, std result_of and std::true_type.
+#include <utility>      // For std::declval.
+
+#ifdef __has_include                      // Check if __has_include is present
+#  if __has_include(<version>)            // Check for version header
+#    include <version>
+#    if defined(__cpp_lib_is_invocable) && !defined(HIP_HAS_INVOCABLE)
+#       define HIP_HAS_INVOCABLE __cpp_lib_is_invocable
+#    endif
+#    if defined(__cpp_lib_result_of_sfinae) && !defined(HIP_HAS_RESULT_OF_SFINAE)
+#       define HIP_HAS_RESULT_OF_SFINAE __cpp_lib_result_of_sfinae
+#    endif
+#  endif
+#endif
+
+#ifndef HIP_HAS_INVOCABLE
+#define HIP_HAS_INVOCABLE 0
+#endif
+
+#ifndef HIP_HAS_RESULT_OF_SFINAE
+#define HIP_HAS_RESULT_OF_SFINAE 0
+#endif
+
+namespace std {  // TODO: these should be removed as soon as possible.
+#if (__cplusplus < 201406L)
+#if (__cplusplus < 201402L)
+template <bool cond, typename T = void>
+using enable_if_t = typename enable_if<cond, T>::type;
+template <bool cond, typename T, typename U>
+using conditional_t = typename conditional<cond, T, U>::type;
+template <typename T>
+using decay_t = typename decay<T>::type;
+template <FunctionalProcedure F, typename... Ts>
+using result_of_t = typename result_of<F(Ts...)>::type;
+template <typename T>
+using remove_reference_t = typename remove_reference<T>::type;
+#endif
+#endif
+}  // namespace std
+
+namespace hip_impl {
+template <typename...>
+using void_t_ = void;
+
+#if HIP_HAS_INVOCABLE
+template <typename, typename = void>
+struct is_callable_impl;
+
+template <FunctionalProcedure F, typename... Ts>
+struct is_callable_impl<F(Ts...)> : std::is_invocable<F, Ts...> {};
+#elif HIP_HAS_RESULT_OF_SFINAE
+template <typename, typename = void>
+struct is_callable_impl : std::false_type {};
+
+template <FunctionalProcedure F, typename... Ts>
+struct is_callable_impl<F(Ts...), void_t_<typename std::result_of<F(Ts...)>::type > > : std::true_type {};
+#else
+template <class Base, class T, class Derived>
+auto simple_invoke(T Base::*pmd, Derived&& ref)
+-> decltype(static_cast<Derived&&>(ref).*pmd);
+
+template <class PMD, class Pointer>
+auto simple_invoke(PMD&& pmd, Pointer&& ptr)
+-> decltype((*static_cast<Pointer&&>(ptr)).*static_cast<PMD&&>(pmd));
+
+template <class Base, class T, class Derived>
+auto simple_invoke(T Base::*pmd, const std::reference_wrapper<Derived>& ref)
+-> decltype(ref.get().*pmd);
+
+template <class Base, class T, class Derived, class... Args>
+auto simple_invoke(T Base::*pmf, Derived&& ref, Args&&... args)
+-> decltype((static_cast<Derived&&>(ref).*pmf)(static_cast<Args&&>(args)...));
+
+template <class PMF, class Pointer, class... Args>
+auto simple_invoke(PMF&& pmf, Pointer&& ptr, Args&&... args)
+-> decltype(((*static_cast<Pointer&&>(ptr)).*static_cast<PMF&&>(pmf))(static_cast<Args&&>(args)...));
+
+template <class Base, class T, class Derived, class... Args>
+auto simple_invoke(T Base::*pmf, const std::reference_wrapper<Derived>& ref, Args&&... args)
+-> decltype((ref.get().*pmf)(static_cast<Args&&>(args)...));
+
+template<class F, class... Ts>
+auto simple_invoke(F&& f, Ts&&... xs)
+-> decltype(f(static_cast<Ts&&>(xs)...));
+
+template <typename, typename = void>
+struct is_callable_impl : std::false_type {};
+
+template <FunctionalProcedure F, typename... Ts>
+struct is_callable_impl<F(Ts...), void_t_<decltype(simple_invoke(std::declval<F>(), std::declval<Ts>()...))> >
+    : std::true_type {};
+
+#endif
+
+template <typename Call>
+struct is_callable : is_callable_impl<Call> {};
+
+#define count_macro_args_impl_hip_(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13,     \
+                                   _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25,     \
+                                   _26, _27, _28, _29, _30, _31, _n, ...)                          \
+    _n
+#define count_macro_args_hip_(...)                                                                 \
+    count_macro_args_impl_hip_(, ##__VA_ARGS__, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20,    \
+                               19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,  \
+                               0)
+
+#define overloaded_macro_expand_hip_(macro, arg_cnt) macro##arg_cnt
+#define overload_macro_impl_hip_(macro, arg_cnt) overloaded_macro_expand_hip_(macro, arg_cnt)
+#define overload_macro_hip_(macro, ...)                                                            \
+    overload_macro_impl_hip_(macro, count_macro_args_hip_(__VA_ARGS__))(__VA_ARGS__)
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/hip_assert.h

@@ -0,0 +1,105 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if defined(__clang__) and defined(__HIP__)
+
+// abort
+extern "C" __device__ inline __attribute__((weak))
+void abort() {
+  __builtin_trap();
+}
+
+// The noinline attribute helps encapsulate the printf expansion,
+// which otherwise has a performance impact just by increasing the
+// size of the calling function. Additionally, the weak attribute
+// allows the function to exist as a global although its definition is
+// included in every compilation unit.
+#if defined(_WIN32) || defined(_WIN64)
+extern "C" __device__ __attribute__((noinline)) __attribute__((weak))
+void _wassert(const wchar_t *_msg, const wchar_t *_file, unsigned _line) {
+    // FIXME: Need `wchar_t` support to generate assertion message.
+    __builtin_trap();
+}
+#else /* defined(_WIN32) || defined(_WIN64) */
+extern "C" __device__ __attribute__((noinline)) __attribute__((weak))
+void __assert_fail(const char *assertion,
+                   const char *file,
+                   unsigned int line,
+                   const char *function)
+{
+  const char fmt[] = "%s:%u: %s: Device-side assertion `%s' failed.\n";
+
+  // strlen is not available as a built-in yet, so we create our own
+  // loop in a macro. With a string literal argument, the compiler
+  // usually manages to replace the loop with a constant.
+  //
+  // The macro does not check for null pointer, since all the string
+  // arguments are defined to be constant literals when called from
+  // the assert() macro.
+  //
+  // NOTE: The loop below includes the null terminator in the length
+  // as required by append_string_n().
+#define __hip_get_string_length(LEN, STR)       \
+  do {                                          \
+    const char *tmp = STR;                      \
+    while (*tmp++);                             \
+    LEN = tmp - STR;                            \
+  } while (0)
+
+  auto msg = __ockl_fprintf_stderr_begin();
+  int len = 0;
+  __hip_get_string_length(len, fmt);
+  msg = __ockl_fprintf_append_string_n(msg, fmt, len, 0);
+  __hip_get_string_length(len, file);
+  msg = __ockl_fprintf_append_string_n(msg, file, len, 0);
+  msg = __ockl_fprintf_append_args(msg, 1, line, 0, 0, 0, 0, 0, 0, 0);
+  __hip_get_string_length(len, function);
+  msg = __ockl_fprintf_append_string_n(msg, function, len, 0);
+  __hip_get_string_length(len, assertion);
+  __ockl_fprintf_append_string_n(msg, assertion, len, /* is_last = */ 1);
+
+#undef __hip_get_string_length
+
+  __builtin_trap();
+}
+
+extern "C" __device__ __attribute__((noinline)) __attribute__((weak))
+void __assertfail()
+{
+    // ignore all the args for now.
+    __builtin_trap();
+}
+#endif /* defined(_WIN32) || defined(_WIN64) */
+
+#if defined(NDEBUG)
+#define __hip_assert(COND)
+#else
+#define __hip_assert(COND)                          \
+  do {                                              \
+    if (!(COND))                                    \
+      __builtin_trap();                             \
+  } while (0)
+#endif
+
+#endif // defined(__clang__) and defined(__HIP__)

projects/clr/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h

@@ -0,0 +1,266 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/hip_cooperative_groups_helper.h
+ *
+ *  @brief Device side implementation of cooperative group feature.
+ *
+ *  Defines helper constructs and APIs which aid the types and device API
+ *  wrappers defined within `amd_detail/hip_cooperative_groups.h`.
+ */
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
+
+#if __cplusplus
+#if !defined(__HIPCC_RTC__)
+#include <hip/amd_detail/amd_hip_runtime.h> // threadId, blockId
+#include <hip/amd_detail/amd_device_functions.h>
+#endif
+#if !defined(__align__)
+#define __align__(x) __attribute__((aligned(x)))
+#endif
+
+#if !defined(__CG_QUALIFIER__)
+#define __CG_QUALIFIER__ __device__ __forceinline__
+#endif
+
+#if !defined(__CG_STATIC_QUALIFIER__)
+#define __CG_STATIC_QUALIFIER__ __device__ static __forceinline__
+#endif
+
+#if !defined(_CG_STATIC_CONST_DECL_)
+#define _CG_STATIC_CONST_DECL_ static constexpr
+#endif
+
+using lane_mask = unsigned long long int;
+namespace cooperative_groups {
+
+/* Global scope */
+template <unsigned int size>
+using is_power_of_2 = __hip_internal::integral_constant<bool, (size & (size - 1)) == 0>;
+
+template <unsigned int size>
+using is_valid_wavefront = __hip_internal::integral_constant<bool, size <= 64>;
+
+template <unsigned int size>
+using is_valid_tile_size =
+    __hip_internal::integral_constant<bool, is_power_of_2<size>::value && is_valid_wavefront<size>::value>;
+
+template <typename T>
+using is_valid_type =
+    __hip_internal::integral_constant<bool, __hip_internal::is_integral<T>::value || __hip_internal::is_floating_point<T>::value>;
+
+namespace internal {
+
+/**
+* @brief Enums representing different cooperative group types
+* @note  This enum is only applicable on Linux.
+*
+ */
+typedef enum {
+  cg_invalid,
+  cg_multi_grid,
+  cg_grid,
+  cg_workgroup,
+  cg_tiled_group,
+  cg_coalesced_group
+} group_type;
+/**
+ *  @ingroup CooperativeG
+ *  @{
+ *  This section describes the cooperative groups functions of HIP runtime API.
+ *
+ *  The cooperative groups provides flexible thread parallel programming algorithms, threads
+ *  cooperate and share data to perform collective computations.
+ *
+ *  @note  Cooperative groups feature is implemented on Linux, under developement
+ *  on Windows.
+ *
+ */
+namespace helper {
+/**
+ * @brief Create output mask from input_mask at places where base_mask is set
+ *
+ * Example: base_mask = 0101'0101, input_mask = 1111'0000
+ * Output mask: 1100
+ * Explaination:
+ *           | | | |  | | | |
+ * base:    0|1|0|1|'0|1|0|1|   // Which bits are set
+ * input:   1|1|1|1|'0|0|0|0|   // Which values are picked
+ *           | | | |  | | | |
+ * output:    1   1    0   0
+ */
+__CG_STATIC_QUALIFIER__ unsigned long long adjust_mask(
+    unsigned long long base_mask, unsigned long long input_mask) {
+  unsigned long long out = 0;
+  for (unsigned int i = 0, index = 0; i < warpSize; i++) {
+    auto lane_active = base_mask & (1ull << i);
+    if (lane_active) {
+      auto result = input_mask & (1ull << i);
+      out |= ((result ? 1ull : 0ull) << index);
+      index++;
+    }
+  }
+  return out;
+}
+}  // namespace helper
+/**
+ *
+ * @brief  Functionalities related to multi-grid cooperative group type
+ * @note  The following cooperative groups functions are only applicable on Linux.
+ *
+ */
+namespace multi_grid {
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_grids() {
+  return static_cast<__hip_uint32_t>(__ockl_multi_grid_num_grids()); }
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t grid_rank() {
+  return static_cast<__hip_uint32_t>(__ockl_multi_grid_grid_rank()); }
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_threads() { return static_cast<__hip_uint32_t>(__ockl_multi_grid_size()); }
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t thread_rank() {
+  return static_cast<__hip_uint32_t>(__ockl_multi_grid_thread_rank()); }
+
+__CG_STATIC_QUALIFIER__ bool is_valid() { return static_cast<bool>(__ockl_multi_grid_is_valid()); }
+
+__CG_STATIC_QUALIFIER__ void sync() { __ockl_multi_grid_sync(); }
+
+}  // namespace multi_grid
+
+/**
+ *  @brief Functionalities related to grid cooperative group type
+ *  @note  The following cooperative groups functions are only applicable on Linux.
+ */
+namespace grid {
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_threads() {
+  return static_cast<__hip_uint32_t>((blockDim.z * gridDim.z) * (blockDim.y * gridDim.y) *
+                    (blockDim.x * gridDim.x));
+}
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t thread_rank() {
+  // Compute global id of the workgroup to which the current thread belongs to
+  __hip_uint32_t blkIdx = static_cast<__hip_uint32_t>((blockIdx.z * gridDim.y * gridDim.x) +
+                               (blockIdx.y * gridDim.x) + (blockIdx.x));
+
+  // Compute total number of threads being passed to reach current workgroup
+  // within grid
+  __hip_uint32_t num_threads_till_current_workgroup =
+      static_cast<__hip_uint32_t>(blkIdx * (blockDim.x * blockDim.y * blockDim.z));
+
+  // Compute thread local rank within current workgroup
+  __hip_uint32_t local_thread_rank = static_cast<__hip_uint32_t>((threadIdx.z * blockDim.y * blockDim.x) +
+                                          (threadIdx.y * blockDim.x) + (threadIdx.x));
+
+  return (num_threads_till_current_workgroup + local_thread_rank);
+}
+
+__CG_STATIC_QUALIFIER__ bool is_valid() { return static_cast<bool>(__ockl_grid_is_valid()); }
+
+__CG_STATIC_QUALIFIER__ void sync() { __ockl_grid_sync(); }
+
+}  // namespace grid
+
+/**
+ *  @brief Functionalities related to `workgroup` (thread_block in CUDA terminology)
+ *  cooperative group type
+ *  @note  The following cooperative groups functions are only applicable on Linux.
+ */
+namespace workgroup {
+
+__CG_STATIC_QUALIFIER__ dim3 group_index() {
+  return (dim3(static_cast<__hip_uint32_t>(blockIdx.x), static_cast<__hip_uint32_t>(blockIdx.y),
+               static_cast<__hip_uint32_t>(blockIdx.z)));
+}
+
+__CG_STATIC_QUALIFIER__ dim3 thread_index() {
+  return (dim3(static_cast<__hip_uint32_t>(threadIdx.x), static_cast<__hip_uint32_t>(threadIdx.y),
+               static_cast<__hip_uint32_t>(threadIdx.z)));
+}
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t num_threads() {
+  return (static_cast<__hip_uint32_t>(blockDim.x * blockDim.y * blockDim.z));
+}
+
+__CG_STATIC_QUALIFIER__ __hip_uint32_t thread_rank() {
+  return (static_cast<__hip_uint32_t>((threadIdx.z * blockDim.y * blockDim.x) +
+                     (threadIdx.y * blockDim.x) + (threadIdx.x)));
+}
+
+__CG_STATIC_QUALIFIER__ bool is_valid() {
+  return true;
+}
+
+__CG_STATIC_QUALIFIER__ void sync() { __syncthreads(); }
+
+__CG_STATIC_QUALIFIER__ dim3 block_dim() {
+  return (dim3(static_cast<__hip_uint32_t>(blockDim.x), static_cast<__hip_uint32_t>(blockDim.y),
+          static_cast<__hip_uint32_t>(blockDim.z)));
+}
+
+}  // namespace workgroup
+
+namespace tiled_group {
+
+// enforce ordering for memory intructions
+__CG_STATIC_QUALIFIER__ void sync() { __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent"); }
+
+}  // namespace tiled_group
+
+namespace coalesced_group {
+
+// enforce ordering for memory intructions
+__CG_STATIC_QUALIFIER__ void sync() { __builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent"); }
+
+// Masked bit count
+//
+// For each thread, this function returns the number of active threads which
+// have i-th bit of x set and come before the current thread.
+__CG_STATIC_QUALIFIER__ unsigned int masked_bit_count(lane_mask x, unsigned int add = 0) {
+  unsigned int counter=0;
+  if (static_cast<int>(warpSize) == 32) {
+    counter = __builtin_amdgcn_mbcnt_lo(static_cast<unsigned int>(x), add);
+  } else {
+    unsigned int lo = static_cast<unsigned int>(x & 0xFFFFFFFF);
+    unsigned int hi = static_cast<unsigned int>((x >> 32) & 0xFFFFFFFF);
+    counter = __builtin_amdgcn_mbcnt_lo(lo, add);
+    counter = __builtin_amdgcn_mbcnt_hi(hi, counter);
+  }
+
+  return counter;
+}
+
+}  // namespace coalesced_group
+
+
+}  // namespace internal
+
+}  // namespace cooperative_groups
+/**
+*  @}
+*/
+
+#endif  // __cplusplus
+#endif  // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H

projects/clr/hipamd/include/hip/amd_detail/hip_fp16_gcc.h

@@ -0,0 +1,260 @@
+#pragma once
+
+#if defined(__cplusplus)
+    #include <cstring>
+#endif
+
+struct __half_raw {
+    unsigned short x;
+};
+
+struct __half2_raw {
+    unsigned short x;
+    unsigned short y;
+};
+
+#if defined(__cplusplus)
+    struct __half;
+
+    __half __float2half(float);
+    float __half2float(__half);
+
+    // BEGIN STRUCT __HALF
+    struct __half {
+    protected:
+        unsigned short __x;
+    public:
+        // CREATORS
+        __half() = default;
+        __half(const __half_raw& x) : __x{x.x} {}
+        #if !defined(__HIP_NO_HALF_CONVERSIONS__)
+            __half(float x) : __x{__float2half(x).__x} {}
+            __half(double x) : __x{__float2half(x).__x} {}
+        #endif
+        __half(const __half&) = default;
+        __half(__half&&) = default;
+        ~__half() = default;
+
+        // MANIPULATORS
+        __half& operator=(const __half&) = default;
+        __half& operator=(__half&&) = default;
+        __half& operator=(const __half_raw& x) { __x = x.x; return *this; }
+        #if !defined(__HIP_NO_HALF_CONVERSIONS__)
+            __half& operator=(float x)
+            {
+                __x = __float2half(x).__x;
+                return *this;
+            }
+            __half& operator=(double x)
+            {
+                return *this = static_cast<float>(x);
+            }
+        #endif
+
+        // ACCESSORS
+        operator float() const { return __half2float(*this); }
+        operator __half_raw() const { return __half_raw{__x}; }
+    };
+    // END STRUCT __HALF
+
+    // BEGIN STRUCT __HALF2
+    struct __half2 {
+    public:
+        __half x;
+        __half y;
+
+        // CREATORS
+        __half2() = default;
+        __half2(const __half2_raw& ix)
+            :
+            x{reinterpret_cast<const __half&>(ix.x)},
+            y{reinterpret_cast<const __half&>(ix.y)}
+        {}
+        __half2(const __half& ix, const __half& iy) : x{ix}, y{iy} {}
+        __half2(const __half2&) = default;
+        __half2(__half2&&) = default;
+        ~__half2() = default;
+
+        // MANIPULATORS
+        __half2& operator=(const __half2&) = default;
+        __half2& operator=(__half2&&) = default;
+        __half2& operator=(const __half2_raw& ix)
+        {
+            x = reinterpret_cast<const __half_raw&>(ix.x);
+            y = reinterpret_cast<const __half_raw&>(ix.y);
+            return *this;
+        }
+
+        // ACCESSORS
+        operator __half2_raw() const
+        {
+            return __half2_raw{
+                reinterpret_cast<const unsigned short&>(x),
+                reinterpret_cast<const unsigned short&>(y)};
+        }
+    };
+    // END STRUCT __HALF2
+
+    inline
+    unsigned short __internal_float2half(
+        float flt, unsigned int& sgn, unsigned int& rem)
+    {
+        unsigned int x{};
+        std::memcpy(&x, &flt, sizeof(flt));
+
+        unsigned int u = (x & 0x7fffffffU);
+        sgn = ((x >> 16) & 0x8000U);
+
+        // NaN/+Inf/-Inf
+        if (u >= 0x7f800000U) {
+            rem = 0;
+            return static_cast<unsigned short>(
+                (u == 0x7f800000U) ? (sgn | 0x7c00U) : 0x7fffU);
+        }
+        // Overflows
+        if (u > 0x477fefffU) {
+            rem = 0x80000000U;
+            return static_cast<unsigned short>(sgn | 0x7bffU);
+        }
+        // Normal numbers
+        if (u >= 0x38800000U) {
+            rem = u << 19;
+            u -= 0x38000000U;
+            return static_cast<unsigned short>(sgn | (u >> 13));
+        }
+        // +0/-0
+        if (u < 0x33000001U) {
+            rem = u;
+            return static_cast<unsigned short>(sgn);
+        }
+        // Denormal numbers
+        unsigned int exponent = u >> 23;
+        unsigned int mantissa = (u & 0x7fffffU);
+        unsigned int shift = 0x7eU - exponent;
+        mantissa |= 0x800000U;
+        rem = mantissa << (32 - shift);
+        return static_cast<unsigned short>(sgn | (mantissa >> shift));
+    }
+
+    inline
+    __half __float2half(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+        if (rem > 0x80000000U || (rem == 0x80000000U && (r.x & 0x1))) ++r.x;
+
+        return r;
+    }
+
+    inline
+    __half __float2half_rn(float x) { return __float2half(x); }
+
+    inline
+    __half __float2half_rz(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+
+        return r;
+    }
+
+    inline
+    __half __float2half_rd(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+        if (rem && sgn) ++r.x;
+
+        return r;
+    }
+
+    inline
+    __half __float2half_ru(float x)
+    {
+        __half_raw r;
+        unsigned int sgn{};
+        unsigned int rem{};
+        r.x = __internal_float2half(x, sgn, rem);
+        if (rem && !sgn) ++r.x;
+
+        return r;
+    }
+
+    inline
+    __half2 __float2half2_rn(float x)
+    {
+        return __half2{__float2half_rn(x), __float2half_rn(x)};
+    }
+
+    inline
+    __half2 __floats2half2_rn(float x, float y)
+    {
+        return __half2{__float2half_rn(x), __float2half_rn(y)};
+    }
+
+    inline
+    float __internal_half2float(unsigned short x)
+    {
+        unsigned int sign = ((x >> 15) & 1);
+        unsigned int exponent = ((x >> 10) & 0x1f);
+        unsigned int mantissa = ((x & 0x3ff) << 13);
+
+        if (exponent == 0x1fU) { /* NaN or Inf */
+            mantissa = (mantissa ? (sign = 0, 0x7fffffU) : 0);
+            exponent = 0xffU;
+        } else if (!exponent) { /* Denorm or Zero */
+            if (mantissa) {
+                unsigned int msb;
+                exponent = 0x71U;
+                do {
+                    msb = (mantissa & 0x400000U);
+                    mantissa <<= 1; /* normalize */
+                    --exponent;
+                } while (!msb);
+                mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
+            }
+        } else {
+            exponent += 0x70U;
+        }
+        unsigned int u = ((sign << 31) | (exponent << 23) | mantissa);
+        float f;
+        memcpy(&f, &u, sizeof(u));
+
+        return f;
+    }
+
+    inline
+    float __half2float(__half x)
+    {
+        return __internal_half2float(static_cast<__half_raw>(x).x);
+    }
+    inline
+    float2 __half22float2(__half2 x)
+    {
+        return float2{__internal_half2float(static_cast<__half2_raw>(x).x),
+                      __internal_half2float(static_cast<__half2_raw>(x).x)};
+    }
+
+    inline
+    float __low2float(__half2 x)
+    {
+        return __internal_half2float(static_cast<__half2_raw>(x).x);
+    }
+
+    inline
+    float __high2float(__half2 x)
+    {
+        return __internal_half2float(static_cast<__half2_raw>(x).y);
+    }
+
+    #if !defined(HIP_NO_HALF)
+        using half = __half;
+        using half2 = __half2;
+    #endif
+#endif // defined(__cplusplus)

projects/clr/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h

@@ -0,0 +1,96 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+// /*
+// Half Math Functions
+// */
+#if !defined(__HIPCC_RTC__)
+#include "host_defines.h"
+#endif
+#ifndef __CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+extern "C"
+{
+    __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
+    __device__ _Float16 __ocml_cos_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_exp_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_exp10_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_exp2_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_floor_f16(_Float16);
+    __device__ __attribute__((const))
+    _Float16 __ocml_fma_f16(_Float16, _Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fabs_f16(_Float16);
+    __device__ __attribute__((const)) int __ocml_isinf_f16(_Float16);
+    __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
+    __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
+    __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
+    __device__ _Float16 __ocml_sin_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fmax_f16(_Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fmin_f16(_Float16, _Float16);
+
+    typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
+    typedef short __2i16 __attribute__((ext_vector_type(2)));
+
+    #if defined(__clang__) && defined(__HIP__)
+    __device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b, float c, bool s);
+    #endif
+
+    __device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16);
+    __device__ __2f16 __ocml_cos_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_exp_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_exp10_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_exp2_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_floor_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_fma_2f16(__2f16, __2f16, __2f16);
+    __device__ __attribute__((const)) __2i16 __ocml_isinf_2f16(__2f16);
+    __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16);
+    __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
+    __device__ __2f16 __ocml_sin_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16);
+    __device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16);
+
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
+
+}
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+//TODO: remove these after they get into clang header __clang_hip_libdevice_declares.h'
+extern "C" {
+    __device__ __attribute__((const)) _Float16 __ocml_fmax_f16(_Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_fmin_f16(_Float16, _Float16);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtp_f16_f32(float);
+    __device__ __attribute__((const)) _Float16 __ocml_cvtrtz_f16_f32(float);
+}

projects/clr/hipamd/include/hip/amd_detail/hip_ldg.h

@@ -0,0 +1,100 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H
+
+#if __HIP_CLANG_ONLY__
+#include "amd_hip_vector_types.h"
+#include "host_defines.h"
+
+__device__ inline static char __ldg(const char* ptr) { return *ptr; }
+
+__device__ inline static char2 __ldg(const char2* ptr) { return *ptr; }
+
+__device__ inline static char4 __ldg(const char4* ptr) { return *ptr; }
+
+__device__ inline static signed char __ldg(const signed char* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned char __ldg(const unsigned char* ptr) { return ptr[0]; }
+
+
+__device__ inline static short __ldg(const short* ptr) { return ptr[0]; }
+
+__device__ inline static short2 __ldg(const short2* ptr) { return ptr[0]; }
+
+__device__ inline static short4 __ldg(const short4* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned short __ldg(const unsigned short* ptr) { return ptr[0]; }
+
+
+__device__ inline static int __ldg(const int* ptr) { return ptr[0]; }
+
+__device__ inline static int2 __ldg(const int2* ptr) { return ptr[0]; }
+
+__device__ inline static int4 __ldg(const int4* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned int __ldg(const unsigned int* ptr) { return ptr[0]; }
+
+
+__device__ inline static long __ldg(const long* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned long __ldg(const unsigned long* ptr) { return ptr[0]; }
+
+
+__device__ inline static long long __ldg(const long long* ptr) { return ptr[0]; }
+
+__device__ inline static longlong2 __ldg(const longlong2* ptr) { return ptr[0]; }
+
+__device__ inline static unsigned long long __ldg(const unsigned long long* ptr) { return ptr[0]; }
+
+
+__device__ inline static uchar2 __ldg(const uchar2* ptr) { return ptr[0]; }
+
+__device__ inline static uchar4 __ldg(const uchar4* ptr) { return ptr[0]; }
+
+
+__device__ inline static ushort2 __ldg(const ushort2* ptr) { return ptr[0]; }
+
+
+__device__ inline static uint2 __ldg(const uint2* ptr) { return ptr[0]; }
+
+__device__ inline static uint4 __ldg(const uint4* ptr) { return ptr[0]; }
+
+
+__device__ inline static ulonglong2 __ldg(const ulonglong2* ptr) { return ptr[0]; }
+
+
+__device__ inline static float __ldg(const float* ptr) { return ptr[0]; }
+
+__device__ inline static float2 __ldg(const float2* ptr) { return ptr[0]; }
+
+__device__ inline static float4 __ldg(const float4* ptr) { return ptr[0]; }
+
+
+__device__ inline static double __ldg(const double* ptr) { return ptr[0]; }
+
+__device__ inline static double2 __ldg(const double2* ptr) { return ptr[0]; }
+
+#endif  // __HIP_CLANG_ONLY__
+
+#endif  // HIP_LDG_H

projects/clr/hipamd/include/hip/amd_detail/hip_runtime_prof.h

@@ -0,0 +1,78 @@
+/*
+Copyright (c) 2019 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
+
+// HIP ROCclr Op IDs enumeration
+enum HipVdiOpId {
+  kHipVdiOpIdDispatch = 0,
+  kHipVdiOpIdCopy     = 1,
+  kHipVdiOpIdBarrier  = 2,
+  kHipVdiOpIdNumber   = 3
+};
+
+// Types of ROCclr commands
+enum HipVdiCommandKind {
+  kHipVdiCommandKernel            = 0x11F0,
+  kHipVdiCommandTask              = 0x11F1,
+  kHipVdiMemcpyDeviceToHost       = 0x11F3,
+  kHipHipVdiMemcpyHostToDevice    = 0x11F4,
+  kHipVdiMemcpyDeviceToDevice     = 0x11F5,
+  kHipVidMemcpyDeviceToHostRect   = 0x1201,
+  kHipVdiMemcpyHostToDeviceRect   = 0x1202,
+  kHipVdiMemcpyDeviceToDeviceRect = 0x1203,
+  kHipVdiFillMemory               = 0x1207,
+};
+
+/**
+ * @brief Initializes activity callback
+ *
+ * @param [input] id_callback Event ID callback function
+ * @param [input] op_callback Event operation callback function
+ * @param [input] arg         Arguments passed into callback
+ *
+ * @returns None
+ */
+void hipInitActivityCallback(void* id_callback, void* op_callback, void* arg);
+
+/**
+ * @brief Enables activity callback
+ *
+ * @param [input] op      Operation, which will trigger a callback (@see HipVdiOpId)
+ * @param [input] enable  Enable state for the callback
+ *
+ * @returns True if successful
+ */
+bool hipEnableActivityCallback(uint32_t op, bool enable);
+
+/**
+ * @brief Returns the description string for the operation kind
+ *
+ * @param [input] id      Command kind id (@see HipVdiCommandKind)
+ *
+ * @returns A pointer to a const string with the command description
+ */
+const char* hipGetCmdName(uint32_t id);
+
+#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
+

projects/clr/hipamd/include/hip/amd_detail/host_defines.h

@@ -0,0 +1,269 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ *  @file  amd_detail/host_defines.h
+ *  @brief TODO-doc
+ */
+
+#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H
+#define HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H
+
+// Add guard to Generic Grid Launch method
+#ifndef GENERIC_GRID_LAUNCH
+#define GENERIC_GRID_LAUNCH 1
+#endif
+
+#if defined(__cplusplus)
+namespace __hip_internal {
+typedef unsigned char uint8_t;
+typedef unsigned short uint16_t;
+typedef unsigned int uint32_t;
+typedef unsigned long long uint64_t;
+typedef signed char int8_t;
+typedef signed short int16_t;
+typedef signed int int32_t;
+typedef signed long long int64_t;
+#if defined(_MSC_VER)
+typedef unsigned long long size_t;
+#else
+typedef unsigned long size_t;
+#endif
+
+template <class _Tp, _Tp __v> struct integral_constant {
+  static constexpr const _Tp value = __v;
+  typedef _Tp value_type;
+  typedef integral_constant type;
+  constexpr operator value_type() const { return value; }
+  constexpr value_type operator()() const { return value; }
+};
+template <class _Tp, _Tp __v> constexpr const _Tp integral_constant<_Tp, __v>::value;
+
+typedef integral_constant<bool, true> true_type;
+typedef integral_constant<bool, false> false_type;
+
+template <bool B> using bool_constant = integral_constant<bool, B>;
+typedef bool_constant<true> true_type;
+typedef bool_constant<false> false_type;
+
+template <bool __B, class __T = void> struct enable_if {};
+template <class __T> struct enable_if<true, __T> { typedef __T type; };
+
+template<bool _B> struct true_or_false_type : public false_type {};
+template<> struct true_or_false_type<true> : public true_type {};
+
+template <class _Tp> struct is_integral : public false_type {};
+template <> struct is_integral<bool> : public true_type {};
+template <> struct is_integral<char> : public true_type {};
+template <> struct is_integral<signed char> : public true_type {};
+template <> struct is_integral<unsigned char> : public true_type {};
+template <> struct is_integral<wchar_t> : public true_type {};
+template <> struct is_integral<short> : public true_type {};
+template <> struct is_integral<unsigned short> : public true_type {};
+template <> struct is_integral<int> : public true_type {};
+template <> struct is_integral<unsigned int> : public true_type {};
+template <> struct is_integral<long> : public true_type {};
+template <> struct is_integral<unsigned long> : public true_type {};
+template <> struct is_integral<long long> : public true_type {};
+template <> struct is_integral<unsigned long long> : public true_type {};
+
+template <class _Tp> struct is_arithmetic : public false_type {};
+template <> struct is_arithmetic<bool> : public true_type {};
+template <> struct is_arithmetic<char> : public true_type {};
+template <> struct is_arithmetic<signed char> : public true_type {};
+template <> struct is_arithmetic<unsigned char> : public true_type {};
+template <> struct is_arithmetic<wchar_t> : public true_type {};
+template <> struct is_arithmetic<short> : public true_type {};
+template <> struct is_arithmetic<unsigned short> : public true_type {};
+template <> struct is_arithmetic<int> : public true_type {};
+template <> struct is_arithmetic<unsigned int> : public true_type {};
+template <> struct is_arithmetic<long> : public true_type {};
+template <> struct is_arithmetic<unsigned long> : public true_type {};
+template <> struct is_arithmetic<long long> : public true_type {};
+template <> struct is_arithmetic<unsigned long long> : public true_type {};
+template <> struct is_arithmetic<float> : public true_type {};
+template <> struct is_arithmetic<double> : public true_type {};
+
+template<typename _Tp> struct is_floating_point : public false_type {};
+template<> struct is_floating_point<float> : public true_type {};
+template<> struct is_floating_point<double> : public true_type {};
+template<> struct is_floating_point<long double> : public true_type {};
+
+template <typename __T, typename __U> struct is_same : public false_type {};
+template <typename __T> struct is_same<__T, __T> : public true_type {};
+
+template<typename _Tp, bool = is_arithmetic<_Tp>::value>
+  struct is_signed : public false_type {};
+template<typename _Tp>
+  struct is_signed<_Tp, true> : public true_or_false_type<_Tp(-1) < _Tp(0)> {};
+
+template<class T>
+    auto test_returnable(int) -> decltype(
+        void(static_cast<T(*)()>(nullptr)), true_type{});
+template<class>
+    auto test_returnable(...) -> false_type;
+
+template<class T>
+    struct type_identity { using type = T; };
+
+template<class T> // Note that `cv void&` is a substitution failure
+    auto try_add_lvalue_reference(int) -> type_identity<T&>;
+template<class T> // Handle T = cv void case
+    auto try_add_lvalue_reference(...) -> type_identity<T>;
+
+template<class T>
+    auto try_add_rvalue_reference(int) -> type_identity<T&&>;
+template<class T>
+    auto try_add_rvalue_reference(...) -> type_identity<T>;
+
+template<class T>
+struct add_lvalue_reference
+    : decltype(try_add_lvalue_reference<T>(0)) {};
+
+template<class T>
+struct add_rvalue_reference
+    : decltype(try_add_rvalue_reference<T>(0)) {};
+
+template<typename T>
+typename add_rvalue_reference<T>::type declval() noexcept;
+
+template<class From, class To>
+    auto test_implicitly_convertible(int) -> decltype(
+        void(declval<void(&)(To)>()(declval<From>())), true_type{});
+
+template<class, class>
+    auto test_implicitly_convertible(...) -> false_type;
+
+template<class T> struct remove_cv { typedef T type; };
+template<class T> struct remove_cv<const T> { typedef T type; };
+template<class T> struct remove_cv<volatile T> { typedef T type; };
+template<class T> struct remove_cv<const volatile T> { typedef T type; };
+
+template<class T>
+struct is_void : public is_same<void, typename remove_cv<T>::type> {};
+
+template<class From, class To>
+struct is_convertible : public integral_constant<bool,
+    (decltype(test_returnable<To>(0))::value &&
+     decltype(test_implicitly_convertible<From, To>(0))::value) ||
+    (is_void<From>::value && is_void<To>::value)> {};
+
+template<typename _CharT> struct char_traits;
+template<typename _CharT, typename _Traits = char_traits<_CharT>> class basic_istream;
+template<typename _CharT, typename _Traits = char_traits<_CharT>> class basic_ostream;
+typedef basic_istream<char> istream;
+typedef basic_ostream<char> ostream;
+
+template<typename _Tp>
+    struct is_standard_layout
+    : public integral_constant<bool, __is_standard_layout(_Tp)>
+    { };
+
+template<typename _Tp>
+    struct is_trivial
+    : public integral_constant<bool, __is_trivial(_Tp)>
+    { };
+
+
+template <bool B, class T, class F> struct conditional { using type = T; };
+template <class T, class F> struct conditional<false, T, F> { using type = F; };
+
+template<class T>
+struct alignment_of : integral_constant<size_t, alignof(T)> {};
+
+template<typename T, T... Ints>
+struct integer_sequence {
+    using value_type = T;
+    static constexpr size_t size() noexcept { return sizeof...(Ints); }
+};
+
+template<size_t... Ints>
+using index_sequence = integer_sequence<size_t, Ints...>;
+
+template<size_t N, size_t... Ints>
+struct make_index_sequence_impl : make_index_sequence_impl<N - 1, N - 1, Ints...> {};
+
+template<size_t... Ints>
+struct make_index_sequence_impl<0, Ints...> {
+    using type = index_sequence<Ints...>;
+};
+
+template<size_t N>
+using make_index_sequence = typename make_index_sequence_impl<N>::type;
+
+template <size_t... Ints>
+constexpr index_sequence<Ints...> make_index_sequence_value(index_sequence<Ints...>) {
+    return {};
+}
+}
+typedef __hip_internal::uint8_t __hip_uint8_t;
+typedef __hip_internal::uint16_t __hip_uint16_t;
+typedef __hip_internal::uint32_t __hip_uint32_t;
+typedef __hip_internal::uint64_t __hip_uint64_t;
+typedef __hip_internal::int8_t __hip_int8_t;
+typedef __hip_internal::int16_t __hip_int16_t;
+typedef __hip_internal::int32_t __hip_int32_t;
+typedef __hip_internal::int64_t __hip_int64_t;
+#endif // defined(__cplusplus)
+
+#if defined(__clang__) && defined(__HIP__)
+#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+#define __host__ __attribute__((host))
+#define __device__ __attribute__((device))
+#define __global__ __attribute__((global))
+#define __shared__ __attribute__((shared))
+#define __constant__ __attribute__((constant))
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+
+#if !defined(__has_feature) || !__has_feature(cuda_noinline_keyword)
+#define __noinline__ __attribute__((noinline))
+#endif
+
+#define __forceinline__ inline __attribute__((always_inline))
+
+#if __HIP_NO_IMAGE_SUPPORT
+#define __hip_img_chk__ __attribute__((unavailable("The image/texture API not supported on the device")))
+#else
+#define __hip_img_chk__
+#endif
+
+#else
+
+// Non-HCC compiler
+/**
+ * Function and kernel markers
+ */
+#define __host__
+#define __device__
+
+#define __global__
+
+#define __noinline__
+#define __forceinline__ inline
+
+#define __shared__
+#define __constant__
+
+#define __hip_img_chk__
+#endif  // defined(__clang__) && defined(__HIP__)
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/hsa_helpers.hpp

@@ -0,0 +1,102 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#pragma once
+
+#include <hsa/hsa.h>
+
+#include <cstdint>
+#include <functional>
+#include <string>
+
+namespace hip_impl {
+inline void* address(hsa_executable_symbol_t x) {
+    void* r = nullptr;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, &r);
+
+    return r;
+}
+
+inline hsa_agent_t agent(hsa_executable_symbol_t x) {
+    hsa_agent_t r = {};
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_AGENT, &r);
+
+    return r;
+}
+
+inline std::uint32_t group_size(hsa_executable_symbol_t x) {
+    std::uint32_t r = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_GROUP_SEGMENT_SIZE, &r);
+
+    return r;
+}
+
+inline hsa_isa_t isa(hsa_agent_t x) {
+    hsa_isa_t r = {};
+    hsa_agent_iterate_isas(x,
+                           [](hsa_isa_t i, void* o) {
+                               *static_cast<hsa_isa_t*>(o) = i;  // Pick the first.
+
+                               return HSA_STATUS_INFO_BREAK;
+                           },
+                           &r);
+
+    return r;
+}
+
+inline std::uint64_t kernel_object(hsa_executable_symbol_t x) {
+    std::uint64_t r = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &r);
+
+    return r;
+}
+
+inline std::string name(hsa_executable_symbol_t x) {
+    std::uint32_t sz = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH, &sz);
+
+    std::string r(sz, '\0');
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME, &r.front());
+
+    return r;
+}
+
+inline std::uint32_t private_size(hsa_executable_symbol_t x) {
+    std::uint32_t r = 0u;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_PRIVATE_SEGMENT_SIZE, &r);
+
+    return r;
+}
+
+inline std::uint32_t size(hsa_executable_symbol_t x) {
+    std::uint32_t r = 0;
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_SIZE, &r);
+
+    return r;
+}
+
+inline hsa_symbol_kind_t type(hsa_executable_symbol_t x) {
+    hsa_symbol_kind_t r = {};
+    hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &r);
+
+    return r;
+}
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp

@@ -0,0 +1,798 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "concepts.hpp"
+#include "helpers.hpp"
+
+#include "hc.hpp"
+#include "hip/hip_ext.h"
+#include "hip_runtime.h"
+
+#include <functional>
+#include <iostream>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+namespace hip_impl {
+namespace {
+struct New_grid_launch_tag {};
+struct Old_grid_launch_tag {};
+
+template <typename C, typename D>
+class RAII_guard {
+    D dtor_;
+
+   public:
+    RAII_guard() = default;
+
+    RAII_guard(const C& ctor, D dtor) : dtor_{std::move(dtor)} { ctor(); }
+
+    RAII_guard(const RAII_guard&) = default;
+    RAII_guard(RAII_guard&&) = default;
+
+    RAII_guard& operator=(const RAII_guard&) = default;
+    RAII_guard& operator=(RAII_guard&&) = default;
+
+    ~RAII_guard() { dtor_(); }
+};
+
+template <typename C, typename D>
+RAII_guard<C, D> make_RAII_guard(const C& ctor, D dtor) {
+    return RAII_guard<C, D>{ctor, std::move(dtor)};
+}
+
+template <FunctionalProcedure F, typename... Ts>
+using is_new_grid_launch_t = typename std::conditional<is_callable<F(Ts...)>{}, New_grid_launch_tag,
+                                                       Old_grid_launch_tag>::type;
+}  // namespace
+
+// TODO: - dispatch rank should be derived from the domain dimensions passed
+//         in, and not always assumed to be 3;
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> ==
+         {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag, dim3 num_blocks,
+                                                    dim3 dim_blocks, int group_mem_bytes,
+                                                    const hc::accelerator_view& acc_v, K k) {
+    const auto d =
+        hc::extent<3>{num_blocks.z * dim_blocks.z, num_blocks.y * dim_blocks.y,
+                      num_blocks.x * dim_blocks.x}
+            .tile_with_dynamic(dim_blocks.z, dim_blocks.y, dim_blocks.x, group_mem_bytes);
+
+    try {
+        hc::parallel_for_each(acc_v, d, k);
+    } catch (std::exception& ex) {
+        std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl;
+        hip_throw(ex);
+    }
+}
+
+// TODO: these are workarounds, they should be removed.
+
+hc::accelerator_view lock_stream_hip_(hipStream_t&, void*&);
+void print_prelaunch_trace_(const char*, dim3, dim3, int, hipStream_t);
+void unlock_stream_hip_(hipStream_t, void*, const char*, hc::accelerator_view*);
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag,
+                                                                 dim3 num_blocks, dim3 dim_blocks,
+                                                                 int group_mem_bytes,
+                                                                 hipStream_t stream,
+                                                                 const char* kernel_name, K k) {
+    void* lck_stream = nullptr;
+    auto acc_v = lock_stream_hip_(stream, lck_stream);
+    auto stream_guard =
+        make_RAII_guard(std::bind(print_prelaunch_trace_, kernel_name, num_blocks, dim_blocks,
+                                  group_mem_bytes, stream),
+                        std::bind(unlock_stream_hip_, stream, lck_stream, kernel_name, &acc_v));
+
+    try {
+        grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
+                              group_mem_bytes, acc_v, std::move(k));
+    } catch (std::exception& ex) {
+        std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl;
+        hip_throw(ex);
+    }
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> ==
+         {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(Old_grid_launch_tag,
+                                                                   dim3 num_blocks, dim3 dim_blocks,
+                                                                   int group_mem_bytes,
+                                                                   hipStream_t stream, K k) {
+    grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
+                          group_mem_bytes, std::move(stream), std::move(k));
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(
+    Old_grid_launch_tag, dim3 num_blocks, dim3 dim_blocks, int group_mem_bytes, hipStream_t stream,
+    const char* kernel_name, K k) {
+    grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
+                          group_mem_bytes, std::move(stream), kernel_name, std::move(k));
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {Ts...}) inline std::enable_if_t<
+    !std::is_function<K>::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks,
+                                                  int group_mem_bytes, hipStream_t stream,
+                                                  const char* kernel_name, K k) {
+    grid_launch_hip_impl_(is_new_grid_launch_t<K, Ts...>{}, std::move(num_blocks),
+                          std::move(dim_blocks), group_mem_bytes, std::move(stream), kernel_name,
+                          std::move(k));
+}
+
+template <FunctionalProcedure K, typename... Ts>
+requires(Domain<K> == {Ts...}) inline std::enable_if_t<
+    !std::is_function<K>::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks,
+                                                  int group_mem_bytes, hipStream_t stream, K k) {
+    grid_launch_hip_impl_(is_new_grid_launch_t<K, Ts...>{}, std::move(num_blocks),
+                          std::move(dim_blocks), group_mem_bytes, std::move(stream), std::move(k));
+}
+
+// TODO: these are temporary and purposefully noisy and disruptive.
+#define make_kernel_name_hip(k, n)                                                                 \
+    HIP_kernel_functor_name_begin##_##k##_##HIP_kernel_functor_name_end##_##n
+
+#define make_kernel_functor_hip_30(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24, p25, p26, p27)                                   \
+    struct make_kernel_name_hip(function_name, 28) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        std::decay_t<decltype(p25)> _p25_;                                                         \
+        std::decay_t<decltype(p26)> _p26_;                                                         \
+        std::decay_t<decltype(p27)> _p27_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_, _p25_, _p26_, _p27_);                                 \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_29(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24, p25, p26)                                        \
+    struct make_kernel_name_hip(function_name, 27) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        std::decay_t<decltype(p25)> _p25_;                                                         \
+        std::decay_t<decltype(p26)> _p26_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_, _p25_, _p26_);                                        \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_28(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24, p25)                                             \
+    struct make_kernel_name_hip(function_name, 26) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        std::decay_t<decltype(p25)> _p25_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_, _p25_);                                               \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_27(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23, p24)                                                  \
+    struct make_kernel_name_hip(function_name, 25) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        std::decay_t<decltype(p24)> _p24_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_, _p24_);                                                      \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_26(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22, p23)                                                       \
+    struct make_kernel_name_hip(function_name, 24) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        std::decay_t<decltype(p23)> _p23_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_, _p23_);                                                             \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_25(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
+                                   p22)                                                            \
+    struct make_kernel_name_hip(function_name, 23) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        std::decay_t<decltype(p22)> _p22_;                                                         \
+        __attribute__((used, flatten)) void operator()(const hc::tiled_index<3>&) const [[hc]] {   \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_,      \
+                        _p22_);                                                                    \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_24(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21) \
+    struct make_kernel_name_hip(function_name, 22) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        std::decay_t<decltype(p21)> _p21_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_);     \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_23(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20)      \
+    struct make_kernel_name_hip(function_name, 21) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        std::decay_t<decltype(p20)> _p20_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_);            \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_22(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19)           \
+    struct make_kernel_name_hip(function_name, 20) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        std::decay_t<decltype(p19)> _p19_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_);                   \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_21(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17, p18)                \
+    struct make_kernel_name_hip(function_name, 19) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        std::decay_t<decltype(p18)> _p18_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_);                          \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_20(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16, p17)                     \
+    struct make_kernel_name_hip(function_name, 18) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        std::decay_t<decltype(p17)> _p17_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_, _p17_);                                 \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_19(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15, p16)                          \
+    struct make_kernel_name_hip(function_name, 17) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        std::decay_t<decltype(p16)> _p16_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_, _p16_);                                        \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_18(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14, p15)                               \
+    struct make_kernel_name_hip(function_name, 16) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        std::decay_t<decltype(p15)> _p15_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_, _p15_);                                               \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_17(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13, p14)                                    \
+    struct make_kernel_name_hip(function_name, 15) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        std::decay_t<decltype(p14)> _p14_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_, _p14_);                                                      \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_16(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12, p13)                                         \
+    struct make_kernel_name_hip(function_name, 14) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        std::decay_t<decltype(p13)> _p13_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_, _p13_);                                                             \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_15(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11, p12)                                              \
+    struct make_kernel_name_hip(function_name, 13) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        std::decay_t<decltype(p12)> _p12_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_,  \
+                        _p12_);                                                                    \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_14(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10, p11)                                                   \
+    struct make_kernel_name_hip(function_name, 12) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        std::decay_t<decltype(p11)> _p11_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_); \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_13(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9, p10)                                                        \
+    struct make_kernel_name_hip(function_name, 11) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        std::decay_t<decltype(p10)> _p10_;                                                         \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] {                                  \
+            kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_);        \
+        }                                                                                          \
+    }
+#define make_kernel_functor_hip_12(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
+                                   p9)                                                             \
+    struct make_kernel_name_hip(function_name, 10) {                                               \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        std::decay_t<decltype(p9)> _p9_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_); }    \
+    }
+#define make_kernel_functor_hip_11(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8) \
+    struct make_kernel_name_hip(function_name, 9) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        std::decay_t<decltype(p8)> _p8_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_); }          \
+    }
+#define make_kernel_functor_hip_10(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7)     \
+    struct make_kernel_name_hip(function_name, 8) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        std::decay_t<decltype(p7)> _p7_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_); }                \
+    }
+#define make_kernel_functor_hip_9(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6)          \
+    struct make_kernel_name_hip(function_name, 7) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        std::decay_t<decltype(p6)> _p6_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_); }                      \
+    }
+#define make_kernel_functor_hip_8(function_name, kernel_name, p0, p1, p2, p3, p4, p5)              \
+    struct make_kernel_name_hip(function_name, 6) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        std::decay_t<decltype(p5)> _p5_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_); }                            \
+    }
+#define make_kernel_functor_hip_7(function_name, kernel_name, p0, p1, p2, p3, p4)                  \
+    struct make_kernel_name_hip(function_name, 5) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        std::decay_t<decltype(p4)> _p4_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_); }                                  \
+    }
+#define make_kernel_functor_hip_6(function_name, kernel_name, p0, p1, p2, p3)                      \
+    struct make_kernel_name_hip(function_name, 4) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        std::decay_t<decltype(p3)> _p3_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const                                           \
+            [[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_); }                                        \
+    }
+#define make_kernel_functor_hip_5(function_name, kernel_name, p0, p1, p2)                          \
+    struct make_kernel_name_hip(function_name, 3) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        std::decay_t<decltype(p2)> _p2_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_, _p2_); } \
+    }
+#define make_kernel_functor_hip_4(function_name, kernel_name, p0, p1)                              \
+    struct make_kernel_name_hip(function_name, 2) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        std::decay_t<decltype(p1)> _p1_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_); }       \
+    }
+#define fofo(f, n) kernel_prefix_hip##f##kernel_suffix_hip##n
+#define make_kernel_functor_hip_3(function_name, kernel_name, p0)                                  \
+    struct make_kernel_name_hip(function_name, 1) {                                                \
+        std::decay_t<decltype(p0)> _p0_;                                                           \
+        void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_); }             \
+    }
+#define make_kernel_functor_hip_2(function_name, kernel_name)                                      \
+    struct make_kernel_name_hip(function_name, 0) {                                                \
+        void operator()(const hc::tiled_index<3>&)[[hc]] { return kernel_name(hipLaunchParm{}); }  \
+    }
+#define make_kernel_functor_hip_1(...)
+#define make_kernel_functor_hip_0(...)
+#define make_kernel_functor_hip_(...) overload_macro_hip_(make_kernel_functor_hip_, __VA_ARGS__)
+
+
+#define hipLaunchNamedKernelGGL(function_name, kernel_name, num_blocks, dim_blocks,                \
+                                group_mem_bytes, stream, ...)                                      \
+    do {                                                                                           \
+        make_kernel_functor_hip_(function_name, kernel_name, __VA_ARGS__)                          \
+            hip_kernel_functor_impl_{__VA_ARGS__};                                                 \
+        hip_impl::grid_launch_hip_(num_blocks, dim_blocks, group_mem_bytes, stream, #kernel_name,  \
+                                   hip_kernel_functor_impl_);                                      \
+    } while (0)
+
+#define hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...)      \
+    do {                                                                                           \
+        hipLaunchNamedKernelGGL(unnamed, kernel_name, num_blocks, dim_blocks, group_mem_bytes,     \
+                                stream, ##__VA_ARGS__);                                            \
+    } while (0)
+
+#define hipLaunchKernel(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...)         \
+    do {                                                                                           \
+        hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream,           \
+                           hipLaunchParm{}, ##__VA_ARGS__);                                        \
+    } while (0)
+}  // namespace hip_impl

projects/clr/hipamd/include/hip/amd_detail/math_fwd.h

@@ -0,0 +1,698 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include "host_defines.h"
+#include "amd_hip_vector_types.h"  // For Native_vec_
+#endif
+
+#if defined(__cplusplus)
+    extern "C" {
+#endif
+
+// DOT FUNCTIONS
+#if defined(__clang__) && defined(__HIP__)
+__device__
+__attribute__((const))
+int __ockl_sdot2(
+    HIP_vector_base<short, 2>::Native_vec_,
+    HIP_vector_base<short, 2>::Native_vec_,
+    int, bool);
+
+__device__
+__attribute__((const))
+unsigned int __ockl_udot2(
+    HIP_vector_base<unsigned short, 2>::Native_vec_,
+    HIP_vector_base<unsigned short, 2>::Native_vec_,
+    unsigned int, bool);
+
+__device__
+__attribute__((const))
+int __ockl_sdot4(
+    HIP_vector_base<char, 4>::Native_vec_,
+    HIP_vector_base<char, 4>::Native_vec_,
+    int, bool);
+
+__device__
+__attribute__((const))
+unsigned int __ockl_udot4(
+    HIP_vector_base<unsigned char, 4>::Native_vec_,
+    HIP_vector_base<unsigned char, 4>::Native_vec_,
+    unsigned int, bool);
+
+__device__
+__attribute__((const))
+int __ockl_sdot8(int, int, int, bool);
+
+__device__
+__attribute__((const))
+unsigned int __ockl_udot8(unsigned int, unsigned int, unsigned int, bool);
+#endif
+
+#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+// BEGIN FLOAT
+__device__
+__attribute__((const))
+float __ocml_acos_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_acosh_f32(float);
+__device__
+__attribute__((const))
+float __ocml_asin_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_asinh_f32(float);
+__device__
+__attribute__((const))
+float __ocml_atan2_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_atan_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_atanh_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_cbrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_ceil_f32(float);
+__device__
+__attribute__((const))
+__device__
+float __ocml_copysign_f32(float, float);
+__device__
+float __ocml_cos_f32(float);
+__device__
+float __ocml_native_cos_f32(float);
+__device__
+__attribute__((pure))
+__device__
+float __ocml_cosh_f32(float);
+__device__
+float __ocml_cospi_f32(float);
+__device__
+float __ocml_i0_f32(float);
+__device__
+float __ocml_i1_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfc_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfcinv_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfcx_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erf_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_erfinv_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_exp10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_exp10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_exp2_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_exp_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_exp_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_expm1_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fabs_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fdim_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_floor_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fma_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fmax_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_fmin_f32(float, float);
+__device__
+__attribute__((const))
+__device__
+float __ocml_fmod_f32(float, float);
+__device__
+float __ocml_frexp_f32(float, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+float __ocml_hypot_f32(float, float);
+__device__
+__attribute__((const))
+int __ocml_ilogb_f32(float);
+__device__
+__attribute__((const))
+int __ocml_isfinite_f32(float);
+__device__
+__attribute__((const))
+int __ocml_isinf_f32(float);
+__device__
+__attribute__((const))
+int __ocml_isnan_f32(float);
+__device__
+float __ocml_j0_f32(float);
+__device__
+float __ocml_j1_f32(float);
+__device__
+__attribute__((const))
+float __ocml_ldexp_f32(float, int);
+__device__
+float __ocml_lgamma_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_log10_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log1p_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log2_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_log2_f32(float);
+__device__
+__attribute__((const))
+float __ocml_logb_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_log_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_native_log_f32(float);
+__device__
+float __ocml_modf_f32(float, __attribute__((address_space(5))) float*);
+__device__
+__attribute__((const))
+float __ocml_nearbyint_f32(float);
+__device__
+__attribute__((const))
+float __ocml_nextafter_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_len3_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_len4_f32(float, float, float, float);
+__device__
+__attribute__((pure))
+float __ocml_ncdf_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_ncdfinv_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_pow_f32(float, float);
+__device__
+__attribute__((pure))
+float __ocml_pown_f32(float, int);
+__device__
+__attribute__((pure))
+float __ocml_rcbrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_remainder_f32(float, float);
+__device__
+float __ocml_remquo_f32(float, float, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+float __ocml_rhypot_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_rint_f32(float);
+__device__
+__attribute__((const))
+float __ocml_rlen3_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_rlen4_f32(float, float, float, float);
+__device__
+__attribute__((const))
+float __ocml_round_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_rsqrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_scalb_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_scalbn_f32(float, int);
+__device__
+__attribute__((const))
+int __ocml_signbit_f32(float);
+__device__
+float __ocml_sincos_f32(float, __attribute__((address_space(5))) float*);
+__device__
+float __ocml_sincospi_f32(float, __attribute__((address_space(5))) float*);
+__device__
+float __ocml_sin_f32(float);
+__device__
+float __ocml_native_sin_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_sinh_f32(float);
+__device__
+float __ocml_sinpi_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_f32(float);
+__device__
+__attribute__((const))
+float __ocml_native_sqrt_f32(float);
+__device__
+float __ocml_tan_f32(float);
+__device__
+__attribute__((pure))
+float __ocml_tanh_f32(float);
+__device__
+float __ocml_tgamma_f32(float);
+__device__
+__attribute__((const))
+float __ocml_trunc_f32(float);
+__device__
+float __ocml_y0_f32(float);
+__device__
+float __ocml_y1_f32(float);
+
+// BEGIN INTRINSICS
+__device__
+__attribute__((const))
+float __ocml_add_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_add_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_add_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_add_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sub_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_mul_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rte_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rtn_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rtp_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_div_rtz_f32(float, float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rte_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rtn_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rtp_f32(float);
+__device__
+__attribute__((const))
+float __ocml_sqrt_rtz_f32(float);
+__device__
+__attribute__((const))
+float __ocml_fma_rte_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fma_rtn_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fma_rtp_f32(float, float, float);
+__device__
+__attribute__((const))
+float __ocml_fma_rtz_f32(float, float, float);
+// END INTRINSICS
+// END FLOAT
+
+// BEGIN DOUBLE
+__device__
+__attribute__((const))
+double __ocml_acos_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_acosh_f64(double);
+__device__
+__attribute__((const))
+double __ocml_asin_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_asinh_f64(double);
+__device__
+__attribute__((const))
+double __ocml_atan2_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_atan_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_atanh_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_cbrt_f64(double);
+__device__
+__attribute__((const))
+double __ocml_ceil_f64(double);
+__device__
+__attribute__((const))
+double __ocml_copysign_f64(double, double);
+__device__
+double __ocml_cos_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_cosh_f64(double);
+__device__
+double __ocml_cospi_f64(double);
+__device__
+double __ocml_i0_f64(double);
+__device__
+double __ocml_i1_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfc_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfcinv_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfcx_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erf_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_erfinv_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_exp10_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_exp2_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_exp_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_expm1_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fabs_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fdim_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_floor_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fma_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fmax_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_fmin_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_fmod_f64(double, double);
+__device__
+double __ocml_frexp_f64(double, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+double __ocml_hypot_f64(double, double);
+__device__
+__attribute__((const))
+int __ocml_ilogb_f64(double);
+__device__
+__attribute__((const))
+int __ocml_isfinite_f64(double);
+__device__
+__attribute__((const))
+int __ocml_isinf_f64(double);
+__device__
+__attribute__((const))
+int __ocml_isnan_f64(double);
+__device__
+double __ocml_j0_f64(double);
+__device__
+double __ocml_j1_f64(double);
+__device__
+__attribute__((const))
+double __ocml_ldexp_f64(double, int);
+__device__
+double __ocml_lgamma_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log10_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log1p_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log2_f64(double);
+__device__
+__attribute__((const))
+double __ocml_logb_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_log_f64(double);
+__device__
+double __ocml_modf_f64(double, __attribute__((address_space(5))) double*);
+__device__
+__attribute__((const))
+double __ocml_nearbyint_f64(double);
+__device__
+__attribute__((const))
+double __ocml_nextafter_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_len3_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_len4_f64(double, double, double, double);
+__device__
+__attribute__((pure))
+double __ocml_ncdf_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_ncdfinv_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_pow_f64(double, double);
+__device__
+__attribute__((pure))
+double __ocml_pown_f64(double, int);
+__device__
+__attribute__((pure))
+double __ocml_rcbrt_f64(double);
+__device__
+__attribute__((const))
+double __ocml_remainder_f64(double, double);
+__device__
+double __ocml_remquo_f64(
+    double, double, __attribute__((address_space(5))) int*);
+__device__
+__attribute__((const))
+double __ocml_rhypot_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_rint_f64(double);
+__device__
+__attribute__((const))
+double __ocml_rlen3_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_rlen4_f64(double, double, double, double);
+__device__
+__attribute__((const))
+double __ocml_round_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_rsqrt_f64(double);
+__device__
+__attribute__((const))
+double __ocml_scalb_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_scalbn_f64(double, int);
+__device__
+__attribute__((const))
+int __ocml_signbit_f64(double);
+__device__
+double __ocml_sincos_f64(double, __attribute__((address_space(5))) double*);
+__device__
+double __ocml_sincospi_f64(double, __attribute__((address_space(5))) double*);
+__device__
+double __ocml_sin_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_sinh_f64(double);
+__device__
+double __ocml_sinpi_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_f64(double);
+__device__
+double __ocml_tan_f64(double);
+__device__
+__attribute__((pure))
+double __ocml_tanh_f64(double);
+__device__
+double __ocml_tgamma_f64(double);
+__device__
+__attribute__((const))
+double __ocml_trunc_f64(double);
+__device__
+double __ocml_y0_f64(double);
+__device__
+double __ocml_y1_f64(double);
+
+// BEGIN INTRINSICS
+__device__
+__attribute__((const))
+double __ocml_add_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_add_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_add_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_add_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sub_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_mul_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rte_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rtn_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rtp_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_div_rtz_f64(double, double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rte_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rtn_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rtp_f64(double);
+__device__
+__attribute__((const))
+double __ocml_sqrt_rtz_f64(double);
+__device__
+__attribute__((const))
+double __ocml_fma_rte_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fma_rtn_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fma_rtp_f64(double, double, double);
+__device__
+__attribute__((const))
+double __ocml_fma_rtz_f64(double, double, double);
+// END INTRINSICS
+// END DOUBLE
+
+#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
+
+#if defined(__cplusplus)
+    } // extern "C"
+#endif

projects/clr/hipamd/include/hip/amd_detail/ockl_image.h

@@ -0,0 +1,177 @@
+/*
+Copyright (c) 2015 - 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_vector_types.h>
+#endif
+
+extern "C" {
+
+#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(4)))
+
+__device__ float4::Native_vec_ __ockl_image_load_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c);
+
+__device__ float4::Native_vec_ __ockl_image_load_1Db(unsigned int ADDRESS_SPACE_CONSTANT*i, int c);
+
+__device__ float4::Native_vec_ __ockl_image_load_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_load_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f);
+
+__device__ float4::Native_vec_ __ockl_image_load_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l);
+
+__device__ float4::Native_vec_ __ockl_image_load_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l);
+
+__device__ void __ockl_image_store_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l, float4::Native_vec_ p);
+
+__device__ void __ockl_image_store_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l, float4::Native_vec_ p);
+
+__device__ float4::Native_vec_ __ockl_image_sample_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float dx, float dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float dx, float dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_grad_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float4::Native_vec_ dx, float4::Native_vec_ dy);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_sample_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
+
+__device__ float4::Native_vec_ __ockl_image_gather4r_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_gather4g_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_gather4b_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ float4::Native_vec_ __ockl_image_gather4a_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
+
+__device__ int __ockl_image_channel_data_type_1D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_1Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_1Db(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2Dad(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_2Dd(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_3D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_CM(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_data_type_CMa(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_1D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_1Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_1Db(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2Da(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2Dad(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_2Dd(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_3D(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_CM(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+__device__ int __ockl_image_channel_order_CMa(unsigned int ADDRESS_SPACE_CONSTANT* i);
+
+}

projects/clr/hipamd/include/hip/amd_detail/program_state.hpp

@@ -0,0 +1,107 @@
+/*
+Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include <hsa/amd_hsa_kernel_code.h>
+#include <hsa/hsa.h>
+#include <hsa/hsa_ext_amd.h>
+#include <hsa/hsa_ven_amd_loader.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+
+#include <hip/hip_common.h>
+
+struct ihipModuleSymbol_t;
+using hipFunction_t = ihipModuleSymbol_t*;
+
+namespace hip_impl {
+
+// This section contains internal APIs that
+// needs to be exported
+#ifdef __GNUC__
+#pragma GCC visibility push (default)
+#endif
+
+struct kernarg_impl;
+class kernarg {
+public:
+    kernarg();
+    kernarg(kernarg&&);
+    ~kernarg();
+    std::uint8_t* data();
+    std::size_t   size();
+    void reserve(std::size_t);
+    void resize(std::size_t);
+private:
+    kernarg_impl* impl;
+};
+
+class kernargs_size_align;
+class program_state_impl;
+class program_state {
+public:
+    program_state();
+    ~program_state();
+    program_state(const program_state&) = delete;
+
+    hipFunction_t kernel_descriptor(std::uintptr_t,
+                                    hsa_agent_t);
+
+    kernargs_size_align get_kernargs_size_align(std::uintptr_t);
+    hsa_executable_t load_executable(const char*, const size_t,
+                                     hsa_executable_t,
+                                     hsa_agent_t);
+    hsa_executable_t load_executable_no_copy(const char*, const size_t,
+                                             hsa_executable_t,
+                                             hsa_agent_t);
+
+    void* global_addr_by_name(const char* name);
+
+private:
+    friend class agent_globals_impl;
+    program_state_impl* impl;
+};
+
+class kernargs_size_align {
+public:
+    std::size_t size(std::size_t n) const;
+    std::size_t alignment(std::size_t n) const;
+    const void* getHandle() const {return handle;};
+private:
+    const void* handle;
+    friend kernargs_size_align program_state::get_kernargs_size_align(std::uintptr_t);
+};
+
+#ifdef __GNUC__
+#pragma GCC visibility pop
+#endif
+
+inline
+__attribute__((visibility("hidden")))
+program_state& get_program_state() {
+    static program_state ps;
+    return ps;
+}
+}  // Namespace hip_impl.

projects/clr/hipamd/include/hip/amd_detail/texture_fetch_functions.h

@@ -0,0 +1,515 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if defined(__cplusplus)
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_vector_types.h>
+#include <hip/hip_texture_types.h>
+#include <hip/amd_detail/ockl_image.h>
+#include <type_traits>
+#endif // !defined(__HIPCC_RTC__)
+
+#define TEXTURE_PARAMETERS_INIT                                                                     \
+    unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)t.textureObject; \
+    unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
+
+template<typename T>
+struct __hip_is_tex_surf_scalar_channel_type
+{
+    static constexpr bool value =
+        __hip_internal::is_same<T, char>::value ||
+        __hip_internal::is_same<T, unsigned char>::value ||
+        __hip_internal::is_same<T, short>::value ||
+        __hip_internal::is_same<T, unsigned short>::value ||
+        __hip_internal::is_same<T, int>::value ||
+        __hip_internal::is_same<T, unsigned int>::value ||
+        __hip_internal::is_same<T, float>::value;
+};
+
+template<typename T>
+struct __hip_is_tex_surf_channel_type
+{
+    static constexpr bool value =
+        __hip_is_tex_surf_scalar_channel_type<T>::value;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_is_tex_surf_channel_type<HIP_vector_type<T, rank>>
+{
+    static constexpr bool value =
+        __hip_is_tex_surf_scalar_channel_type<T>::value &&
+        ((rank == 1) ||
+         (rank == 2) ||
+         (rank == 4));
+};
+
+template<typename T>
+struct __hip_is_tex_normalized_channel_type
+{
+    static constexpr bool value =
+        __hip_internal::is_same<T, char>::value ||
+        __hip_internal::is_same<T, unsigned char>::value ||
+        __hip_internal::is_same<T, short>::value ||
+        __hip_internal::is_same<T, unsigned short>::value;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_is_tex_normalized_channel_type<HIP_vector_type<T, rank>>
+{
+    static constexpr bool value =
+        __hip_is_tex_normalized_channel_type<T>::value &&
+        ((rank == 1) ||
+         (rank == 2) ||
+         (rank == 4));
+};
+
+template <
+    typename T,
+    hipTextureReadMode readMode,
+    typename Enable = void>
+struct __hip_tex_ret
+{
+    static_assert(__hip_internal::is_same<Enable, void>::value, "Invalid channel type!");
+};
+
+/*
+ * Map from device function return U to scalar texture type T
+ */
+template<typename T, typename U>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+  __hip_is_tex_surf_scalar_channel_type<T>::value, const T>::type
+__hipMapFrom(const U &u) {
+  if constexpr (sizeof(T) < sizeof(float)) {
+    union {
+      U u;
+      int i;
+    } d = { u };
+    return static_cast<T>(d.i);
+  } else { // sizeof(T) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { u };
+    return d.t;
+  }
+}
+
+/*
+ * Map from device function return U to vector texture type T
+ */
+template<typename T, typename U>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+  __hip_is_tex_surf_scalar_channel_type<typename T::value_type>::value, const T>::type
+__hipMapFrom(const U &u) {
+  if constexpr (sizeof(typename T::value_type) < sizeof(float)) {
+    union {
+      U u;
+      int4 i4;
+    } d = { u };
+    return __hipMapVector<typename T::value_type, sizeof(T)/sizeof(typename T::value_type)>(d.i4);
+  } else { // sizeof(typename T::value_type) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { u };
+    return d.t;
+  }
+}
+
+/*
+ * Map from scalar texture type T to device function input U
+ */
+template<typename U, typename T>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+__hip_is_tex_surf_scalar_channel_type<T>::value, const U>::type
+__hipMapTo(const T &t) {
+  if constexpr (sizeof(T) < sizeof(float)) {
+    union {
+      U u;
+      int i;
+    } d = { 0 };
+    d.i = static_cast<int>(t);
+    return d.u;
+  } else { // sizeof(T) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { 0 };
+    d.t = t;
+    return d.u;
+  }
+}
+
+/*
+ * Map from vector texture type T to device function input U
+ */
+template<typename U, typename T>
+__forceinline__ __device__
+typename __hip_internal::enable_if<
+  __hip_is_tex_surf_scalar_channel_type<typename T::value_type>::value, const U>::type
+__hipMapTo(const T &t) {
+  if constexpr (sizeof(typename T::value_type) < sizeof(float)) {
+    union {
+      U u;
+      int4 i4;
+    } d = { 0 };
+    d.i4 = __hipMapVector<int, 4>(t);
+    return d.u;
+  } else { // sizeof(typename T::value_type) == sizeof(float)
+    union {
+      U u;
+      T t;
+    } d = { 0 };
+    d.t = t;
+    return d.u;
+  }
+}
+
+template <
+    typename T,
+    hipTextureReadMode readMode>
+using __hip_tex_ret_t = typename __hip_tex_ret<T, readMode, bool>::type;
+
+template <typename T>
+struct __hip_tex_ret<
+    T,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value, bool>::type>
+{
+    using type = T;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_tex_ret<
+    HIP_vector_type<T, rank>,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
+{
+    using type = HIP_vector_type<__hip_tex_ret_t<T, hipReadModeElementType>, rank>;
+};
+
+template<typename T>
+struct __hip_tex_ret<
+    T,
+    hipReadModeNormalizedFloat,
+    typename __hip_internal::enable_if<__hip_is_tex_normalized_channel_type<T>::value, bool>::type>
+{
+    using type = float;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_tex_ret<
+    HIP_vector_type<T, rank>,
+    hipReadModeNormalizedFloat,
+    typename __hip_internal::enable_if<__hip_is_tex_normalized_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
+{
+    using type = HIP_vector_type<__hip_tex_ret_t<T, hipReadModeNormalizedFloat>, rank>;
+};
+
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1Dfetch(texture<T, hipTextureType1D, readMode> t, int x)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_load_1Db(i, x);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1D(texture<T, hipTextureType1D, readMode> t, float x)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_sample_1D(i, s, x);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2D(texture<T, hipTextureType2D, readMode> t, float x, float y)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_2D(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLayered(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_1Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLayered(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_2Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex3D(texture<T, hipTextureType3D, readMode> t, float x, float y, float z)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_3D(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemap(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_CM(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLod(texture<T, hipTextureType1D, readMode> t, float x, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_sample_lod_1D(i, s, x, level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLod(texture<T, hipTextureType2D, readMode> t, float x, float y, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_lod_2D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLayeredLod(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_lod_1Da(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLayeredLod(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_lod_2Da(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex3DLod(texture<T, hipTextureType3D, readMode> t, float x, float y, float z, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_3D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLod(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_CM(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLayered(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_CMa(i, s, get_native_vector(coords));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLayeredLod(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer, float level)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_lod_CMa(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapGrad(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CM(i, s, get_native_vector(float4(x, y, z, 0.0f)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+    return {};
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> texCubemapLayeredGrad(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CMa(i, s, get_native_vector(float4(x, y, z, layer)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+    return {};
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DGrad(texture<T, hipTextureType1D, readMode> t, float x, float dPdx, float dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DGrad(texture<T, hipTextureType2D, readMode> t, float x, float y, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_grad_2D(i, s, get_native_vector(coords), get_native_vector(dPdx),
+                                           get_native_vector(dPdy));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex1DLayeredGrad(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer, float dPdx, float dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_grad_1Da(i, s, get_native_vector(coords), dPdx, dPdy);
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex2DLayeredGrad(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_grad_2Da(i, s, get_native_vector(coords),
+                                            get_native_vector(dPdx), get_native_vector(dPdy));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex_ret_t<T, readMode> tex3DGrad(texture<T, hipTextureType3D, readMode> t, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float4 coords{x, y, z, 0.0f};
+    float4 gradx{dPdx.x, dPdx.y, dPdx.z, 0.0f};
+    float4 grady{dPdy.x, dPdy.y, dPdy.z, 0.0f};
+    auto tmp = __ockl_image_sample_grad_3D(i, s, get_native_vector(coords),
+                                           get_native_vector(gradx), get_native_vector(grady));
+    return __hipMapFrom<__hip_tex_ret_t<T, readMode>>(tmp);
+}
+
+template <
+    typename T,
+    hipTextureReadMode readMode,
+    typename Enable = void>
+struct __hip_tex2dgather_ret
+{
+    static_assert(__hip_internal::is_same<Enable, void>::value, "Invalid channel type!");
+};
+
+template <
+    typename T,
+    hipTextureReadMode readMode>
+using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret<T, readMode, bool>::type;
+
+template <typename T>
+struct __hip_tex2dgather_ret<
+    T,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value, bool>::type>
+{
+    using type = HIP_vector_type<T, 4>;
+};
+
+template<
+    typename T,
+    unsigned int rank>
+struct __hip_tex2dgather_ret<
+    HIP_vector_type<T, rank>,
+    hipReadModeElementType,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
+{
+    using type = HIP_vector_type<T, 4>;
+};
+
+template <typename T>
+struct __hip_tex2dgather_ret<
+    T,
+    hipReadModeNormalizedFloat,
+    typename __hip_internal::enable_if<__hip_is_tex_normalized_channel_type<T>::value, bool>::type>
+{
+    using type = float4;
+};
+
+template <typename T, hipTextureReadMode readMode>
+static __forceinline__ __device__ __hip_img_chk__ __hip_tex2dgather_ret_t<T, readMode> tex2Dgather(texture<T, hipTextureType2D, readMode> t, float x, float y, int comp=0)
+{
+    TEXTURE_PARAMETERS_INIT;
+    float2 coords{x, y};
+    switch (comp) {
+    case 1: {
+      auto tmp = __ockl_image_gather4g_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    case 2: {
+      auto tmp = __ockl_image_gather4b_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    case 3: {
+      auto tmp = __ockl_image_gather4a_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    default: {
+      auto tmp = __ockl_image_gather4r_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<__hip_tex2dgather_ret_t<T, readMode>>(tmp);
+    }
+    }
+    return {};
+}
+
+#endif

projects/clr/hipamd/include/hip/amd_detail/texture_indirect_functions.h

@@ -0,0 +1,502 @@
+/*
+Copyright (c) 2015 - 2025 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#if defined(__cplusplus)
+
+#if !defined(__HIPCC_RTC__)
+#include <hip/hip_vector_types.h>
+#include <hip/hip_texture_types.h>
+#include <hip/amd_detail/texture_fetch_functions.h>
+#include <hip/amd_detail/ockl_image.h>
+#include <type_traits>
+#endif // !defined(__HIPCC_RTC__)
+
+#define TEXTURE_OBJECT_PARAMETERS_INIT                                                            \
+    unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \
+    unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1Dfetch(hipTextureObject_t textureObject, int x)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_load_1Db(i, x);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1Dfetch(T *ptr, hipTextureObject_t textureObject, int x)
+{
+    *ptr = tex1Dfetch<T>(textureObject, x);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1D(hipTextureObject_t textureObject, float x)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_sample_1D(i, s, x);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1D(T *ptr, hipTextureObject_t textureObject, float x)
+{
+    *ptr = tex1D<T>(textureObject, x);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2D(hipTextureObject_t textureObject, float x, float y)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_2D(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2D(T *ptr, hipTextureObject_t textureObject, float x, float y)
+{
+    *ptr = tex2D<T>(textureObject, x, y);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_3D(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex3D(T *ptr, hipTextureObject_t textureObject, float x, float y, float z)
+{
+    *ptr = tex3D<T>(textureObject, x, y, z);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_1Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLayered(T *ptr, hipTextureObject_t textureObject, float x, int layer)
+{
+    *ptr = tex1DLayered<T>(textureObject, x, layer);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, int layer)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_2Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer)
+{
+    *ptr = tex1DLayered<T>(textureObject, x, y, layer);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__  T texCubemap(hipTextureObject_t textureObject, float x, float y, float z)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_CM(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemap(T *ptr, hipTextureObject_t textureObject, float x, float y, float z)
+{
+    *ptr = texCubemap<T>(textureObject, x, y, z);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapLayered(hipTextureObject_t textureObject, float x, float y, float z, int layer)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_CMa(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer)
+{
+    *ptr = texCubemapLayered<T>(textureObject, x, y, z, layer);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2Dgather(hipTextureObject_t textureObject, float x, float y, int comp = 0)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    switch (comp) {
+    case 1: {
+      auto tmp = __ockl_image_gather4r_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    case 2: {
+      auto tmp = __ockl_image_gather4g_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    case 3: {
+      auto tmp = __ockl_image_gather4b_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    default: {
+      auto tmp = __ockl_image_gather4a_2D(i, s, get_native_vector(coords));
+      return __hipMapFrom<T>(tmp);
+      break;
+    }
+    }
+    return {};
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2Dgather(T *ptr, hipTextureObject_t textureObject, float x, float y, int comp = 0)
+{
+    *ptr = texCubemapLayered<T>(textureObject, x, y, comp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLod(hipTextureObject_t textureObject, float x, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_sample_lod_1D(i, s, x, level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLod(T *ptr, hipTextureObject_t textureObject, float x, float level)
+{
+    *ptr = tex1DLod<T>(textureObject, x, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_lod_2D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float level)
+{
+    *ptr = tex2DLod<T>(textureObject, x, y, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_3D(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex3DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    *ptr = tex3DLod<T>(textureObject, x, y, z, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_1Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, int layer, float level)
+{
+    *ptr = tex1DLayeredLod<T>(textureObject, x, layer, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__  T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, int layer, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_2Da(i, s, get_native_vector(coords));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float level)
+{
+    *ptr = tex2DLayeredLod<T>(textureObject, x, y, layer, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapLod(hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    auto tmp = __ockl_image_sample_lod_CM(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level)
+{
+    *ptr = texCubemapLod<T>(textureObject, x, y, z, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CM(i, s, get_native_vector(float4(x, y, z, 0.0f)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<T>(tmp);
+    return {};
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    *ptr = texCubemapGrad<T>(textureObject, x, y, z, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float x, float y, float z, int layer, float level)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, layer};
+    auto tmp = __ockl_image_sample_lod_CMa(i, s, get_native_vector(coords), level);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float level)
+{
+    *ptr = texCubemapLayeredLod<T>(textureObject, x, y, z, layer, level);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dPdx, float dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DGrad(T *ptr, hipTextureObject_t textureObject, float x, float dPdx, float dPdy)
+{
+    *ptr = tex1DGrad<T>(textureObject, x, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, y};
+    auto tmp = __ockl_image_sample_grad_2D(i, s, get_native_vector(coords), get_native_vector(dPdx),
+                                           get_native_vector(dPdy));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy)
+{
+    *ptr = tex2DGrad<T>(textureObject, x, y, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, z, 0.0f};
+    float4 gradx{dPdy.x, dPdy.y, dPdy.z, 0.0f};
+    float4 grady{dPdy.x, dPdy.y, dPdy.z, 0.0f};
+    auto tmp = __ockl_image_sample_grad_3D(i, s, get_native_vector(coords),
+                                           get_native_vector(gradx), get_native_vector(grady));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex3DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
+{
+    *ptr = tex3DGrad<T>(textureObject, x, y, z, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float2 coords{x, layer};
+    auto tmp = __ockl_image_sample_grad_1Da(i, s, get_native_vector(coords), dPdx, dPdy);
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex1DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy)
+{
+    *ptr = tex1DLayeredGrad<T>(textureObject, x, layer, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    float4 coords{x, y, layer, 0.0f};
+    auto tmp = __ockl_image_sample_grad_2Da(i, s, get_native_vector(coords),
+                                            get_native_vector(dPdx), get_native_vector(dPdy));
+    return __hipMapFrom<T>(tmp);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void tex2DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy)
+{
+    *ptr = tex2DLayeredGrad<T>(textureObject, x, y, layer, dPdx, dPdy);
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__  T texCubemapLayeredGrad(hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
+{
+    TEXTURE_OBJECT_PARAMETERS_INIT
+    // TODO missing in device libs.
+    // auto tmp = __ockl_image_sample_grad_CMa(i, s, get_native_vector(float4(x, y, z, layer)),
+    // get_native_vector(float4(dPdx.x, dPdx.y, dPdx.z, 0.0f)), get_native_vector(float4(dPdy.x,
+    // dPdy.y, dPdy.z, 0.0f))); return __hipMapFrom<T>(tmp);
+    return {};
+}
+
+template <
+    typename T,
+    typename __hip_internal::enable_if<__hip_is_tex_surf_channel_type<T>::value>::type* = nullptr>
+static __device__ __hip_img_chk__ void texCubemapLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
+{
+    *ptr = texCubemapLayeredGrad<T>(textureObject, x, y, z, layer, dPdx, dPdy);
+}
+
+#endif

projects/clr/hipamd/packaging/CMakeLists.txt

@@ -0,0 +1,340 @@
+# Copyright (c) 2020 - 2022 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+cmake_minimum_required(VERSION 3.16.8)
+
+#set components for HIP
+set(CPACK_COMPONENTS_ALL binary dev doc runtime-nvidia)
+
+# ASAN Package requires only libraries and license file
+if(ENABLE_ASAN_PACKAGING)
+   set(CPACK_COMPONENTS_ALL asan)
+endif()
+###############Install Required files for all compnents########
+
+#Enable Component Install
+set(CPACK_RPM_COMPONENT_INSTALL ON)
+set(CPACK_DEB_COMPONENT_INSTALL ON)
+
+###Set License####
+set(CPACK_RESOURCE_FILE_LICENSE ${hip_SOURCE_DIR}/LICENSE.txt)
+install(FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT binary)
+# install license file in share/doc/hip-asan folder
+install(FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR}-asan COMPONENT asan)
+set(CPACK_RPM_PACKAGE_LICENSE "MIT")
+#Begin binary files install
+if(HIP_PLATFORM STREQUAL "amd" )
+   # Set component name and  library type
+   set(COMPONENT_NAME "binary")
+   set(CMAKE_LIB_TYPE "LIBRARY")
+
+   if(NOT BUILD_SHARED_LIBS)
+      # Change library type for static builds
+      set(CMAKE_LIB_TYPE "ARCHIVE")
+   endif()
+
+   if(ENABLE_ASAN_PACKAGING)
+      # Change component name for ASAN builds
+      set(COMPONENT_NAME "asan")
+   endif()
+
+   # Install libraries
+   install(TARGETS amdhip64
+              ${CMAKE_LIB_TYPE}
+              DESTINATION ${CMAKE_INSTALL_LIBDIR}
+              COMPONENT ${COMPONENT_NAME})
+   install(TARGETS hiprtc
+              ${CMAKE_LIB_TYPE}
+              DESTINATION ${CMAKE_INSTALL_LIBDIR}
+              COMPONENT ${COMPONENT_NAME})
+   install(TARGETS hiprtc-builtins
+              ${CMAKE_LIB_TYPE}
+              DESTINATION ${CMAKE_INSTALL_LIBDIR}
+              COMPONENT ${COMPONENT_NAME})
+
+#TODO:This do not belong in BINARY package.
+#Keeping it as is for now
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/share/hip/.hipInfo DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT binary)
+
+install ( EXPORT hip-targets  FILE hip-targets.cmake NAMESPACE hip:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip  COMPONENT dev)
+
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/src/hip-lang-config.cmake ${CMAKE_BINARY_DIR}/hipamd/src/hip-lang-config-version.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip-lang COMPONENT dev)
+install ( EXPORT hip-lang-targets  FILE hip-lang-targets.cmake NAMESPACE hip-lang:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip-lang  COMPONENT dev)
+
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/hiprtc-config.cmake ${CMAKE_BINARY_DIR}/hipamd/hiprtc-config-version.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hiprtc COMPONENT dev)
+install ( EXPORT hiprtc-targets  FILE hiprtc-targets.cmake NAMESPACE hiprtc:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hiprtc  COMPONENT dev)
+
+endif()#End HIP_PLATFORM = "amd"
+#End bianry files install
+
+#Begin dev files install
+#Install bin files from HIP_COMMON_DIR
+file(GLOB BIN_FILES ${HIP_COMMON_DIR}/bin/*)
+if(NOT WIN32)
+    list(FILTER BIN_FILES EXCLUDE REGEX ".bat$")
+endif()
+foreach(binFile ${BIN_FILES})
+    install(PROGRAMS ${binFile} DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT dev)
+endforeach()
+unset(BIN_FILES)
+
+#Install bin files from hip_SOURCE_DIR
+file(GLOB BIN_FILES ${hip_SOURCE_DIR}/bin/*)
+if(NOT WIN32)
+    list(FILTER BIN_FILES EXCLUDE REGEX ".bat$")
+endif()
+foreach(binFile ${BIN_FILES})
+    install(PROGRAMS ${binFile} DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT dev)
+endforeach()
+
+install(DIRECTORY ${HIP_COMMON_DIR}/include DESTINATION . COMPONENT dev)
+install(DIRECTORY ${hip_SOURCE_DIR}/include/hip/amd_detail
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip COMPONENT dev)
+if(DEFINED HIPNV_DIR)
+    install(DIRECTORY ${HIPNV_DIR}/include/hip/nvidia_detail
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip COMPONENT dev)
+endif()
+if(HIP_PLATFORM STREQUAL "amd" )
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/include/hip/amd_detail/hip_prof_str.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip/amd_detail COMPONENT dev)
+endif()
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/include/hip/hip_version.h
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip COMPONENT dev)
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/share/hip/version DESTINATION ${CMAKE_INSTALL_DATADIR}/hip COMPONENT dev)
+# .hipVersion is added to satisfy Windows compute build.
+#TODO to be removed
+if(WIN32)
+    install(FILES ${CMAKE_BINARY_DIR}/hipamd/share/hip/version DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME .hipVersion COMPONENT dev)
+endif()
+install(DIRECTORY ${HIP_COMMON_DIR}/cmake/ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip COMPONENT dev)
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/hip-config.cmake ${CMAKE_BINARY_DIR}/hipamd/hip-config-version.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip COMPONENT dev)
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/hip-config-amd.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip COMPONENT dev)
+install(FILES ${CMAKE_BINARY_DIR}/hipamd/hip-config-nvidia.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip COMPONENT dev)
+#End dev files install
+
+#Begin doc files install
+find_program(DOXYGEN_EXE doxygen)
+if(DOXYGEN_EXE)
+    if(EXISTS "${HIP_COMMON_DIR}/docs/doxygen-input/doxy.cfg")
+        add_custom_target(build_doxygen ALL
+            COMMAND HIP_PATH=${HIP_COMMON_DIR} doxygen ${HIP_COMMON_DIR}/docs/doxygen-input/doxy.cfg)
+    elseif(EXISTS "${HIP_COMMON_DIR}/docs/.doxygen/Doxyfile")
+        add_custom_target(build_doxygen ALL
+            COMMAND HIP_PATH=${HIP_COMMON_DIR} doxygen ${HIP_COMMON_DIR}/docs/.doxygen/Doxyfile)
+    else()
+        message(FATAL_ERROR "Unable to find doxygen config file")
+    endif()
+    install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/RuntimeAPI/html
+            DESTINATION ${CMAKE_INSTALL_DOCDIR}/RuntimeAPI COMPONENT doc)
+endif()
+#End doc files install
+
+##################################
+# Packaging steps COMMON Variables
+##################################
+set(CPACK_PACKAGE_VENDOR "Advanced Micro Devices, Inc.")
+set(CPACK_PACKAGE_CONTACT "HIP Support <hip.support@amd.com>")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "HIP:Heterogenous-computing Interface for Portability")
+set(CPACK_PACKAGE_VERSION_MAJOR ${HIP_VERSION_MAJOR})
+set(CPACK_PACKAGE_VERSION_MINOR ${HIP_VERSION_MINOR})
+set(CPACK_PACKAGE_VERSION_PATCH ${HIP_VERSION_PATCH})
+set(CPACK_PACKAGE_VERSION ${HIP_VERSION_MAJOR}.${HIP_VERSION_MINOR}.${HIP_PACKAGING_VERSION_PATCH})
+set(CPACK_GENERATOR "TGZ;DEB;RPM" CACHE STRING "Package types to build")
+
+set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/opt")
+if (CPACK_RPM_PACKAGE_RELEASE MATCHES "local" )
+  #If building locally default value will cause build failure
+  #DEBUG SYMBOL pacaking require SOURCE_DIR to be small
+  set(CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX ${CPACK_INSTALL_PREFIX})
+endif()
+
+# Eventhough hip-runtime package has libraries,it was not in the package provides list,
+# since CPACK_RPM_PACKAGE_AUTOREQPROV was set to "no".
+# Use AUTOREQ,(rather than AUTOREQPROV) so that package will also provides the libraries
+set(CPACK_RPM_PACKAGE_AUTOREQ " no")
+set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
+
+set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT")
+
+set(CPACK_SOURCE_GENERATOR "TGZ")
+
+
+#Begin Binary Packaging setting
+
+set(CPACK_BINARY_DEB "ON")
+set(CPACK_BINARY_RPM "ON")
+
+set(CPACK_DEBIAN_BINARY_PACKAGE_NAME "hip-runtime-amd")
+set(CPACK_RPM_BINARY_PACKAGE_NAME "hip-runtime-amd")
+
+set(CPACK_COMPONENT_BINARY_DESCRIPTION "HIP:Heterogenous-computing Interface for Portability [RUNTIME - AMD]")
+# Set Runtime Package  dependencies
+set(HIP_RUNTIME_ROCM_PKG_DEPENDENCIES "hsa-rocr, rocminfo, comgr, rocm-core")
+# Add rocprofiler-register dependencies
+if(HIP_ENABLE_ROCPROFILER_REGISTER)
+   set(HIP_RUNTIME_ROCM_PKG_DEPENDENCIES "${HIP_RUNTIME_ROCM_PKG_DEPENDENCIES}, rocprofiler-register")
+endif()
+
+set(HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES "libnuma1, libstdc++6, libc6")
+
+if(DEB10_DISTRO)
+  # On Debian Buster only: libgcc_s.so.1 is in the package libgcc1
+  set(HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES "${HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES}, libgcc1")
+else()
+  set(HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES "${HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES}, libgcc-s1")
+endif()
+
+set(CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "${HIP_RUNTIME_ROCM_PKG_DEPENDENCIES}, ${HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES}")
+
+set(CPACK_DEBIAN_BINARY_PACKAGE_PROVIDES "hip-rocclr (= ${CPACK_PACKAGE_VERSION})")
+set(CPACK_DEBIAN_BINARY_PACKAGE_REPLACES "hip-rocclr (= ${CPACK_PACKAGE_VERSION})")
+
+set(CPACK_RPM_BINARY_PACKAGE_ARCHITECTURE "${CMAKE_SYSTEM_PROCESSOR}")
+string(REPLACE "-" "_" HIP_BASE_VERSION ${CPACK_PACKAGE_VERSION})
+
+if(EL7_DISTRO)
+  # centos: In centos using parenthesis is causing error. So set the specific dependencies
+  set(HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES "glibc, numactl-libs, libstdc++, libgcc")
+else()
+  # RHEL/SLES
+  set(HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES "glibc, (numactl-libs or libnuma1), (libstdc++ or libstdc++6)")
+  if(DEB10_DISTRO)
+    # On Debian Buster only: libgcc_s.so.1 is in the package libgcc1
+    set(HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES "${HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES}, libgcc1")
+  else()
+    set(HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES "${HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES}, (libgcc or libgcc_s1)" )
+  endif()
+endif()
+
+set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "${HIP_RUNTIME_ROCM_PKG_DEPENDENCIES}, ${HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES}")
+
+set(CPACK_RPM_BINARY_PACKAGE_PROVIDES "hip-rocclr = ${HIP_BASE_VERSION}")
+set(CPACK_RPM_BINARY_PACKAGE_OBSOLETES "hip-rocclr = ${HIP_BASE_VERSION}")
+#End Binary Packaging setting
+
+#Begin dev Packaging setting
+set(CPACK_DEV_DEB "ON")
+set(CPACK_DEV_RPM "ON")
+
+set(CPACK_DEBIAN_DEV_PACKAGE_NAME "hip-dev")
+set(CPACK_RPM_DEV_PACKAGE_NAME "hip-devel")
+
+set(CPACK_COMPONENT_DEV_DESCRIPTION "HIP: Heterogenous-computing Interface for Portability [DEVELOPMENT]")
+
+configure_file(hip-devel.postinst ${CMAKE_CURRENT_BINARY_DIR}/dev/postinst @ONLY)
+configure_file(hip-devel.prerm    ${CMAKE_CURRENT_BINARY_DIR}/dev/prerm @ONLY)
+set(CPACK_DEBIAN_DEV_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_BINARY_DIR}/dev/postinst;${CMAKE_CURRENT_BINARY_DIR}/dev/prerm")
+# Dev/devel package dependencies
+set(HIP_DEV_ROCM_PKG_DEPENDENCIES "hip-runtime-amd, rocm-llvm, rocm-core")
+set(CPACK_DEBIAN_DEV_PACKAGE_RECOMMENDS "perl (>= 5.0), libfile-copy-recursive-perl, libfile-listing-perl, libfile-which-perl, liburi-perl")
+set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "libc6, file, ${HIP_DEV_ROCM_PKG_DEPENDENCIES}, hsa-rocr-dev, hipcc")
+set(CPACK_DEBIAN_DEV_PACKAGE_PROVIDES "hip-base")
+set(CPACK_DEBIAN_DEV_PACKAGE_REPLACES "hip-base")
+
+set(CPACK_RPM_DEV_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/dev/postinst")
+set(CPACK_RPM_DEV_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/dev/prerm")
+set(CPACK_RPM_DEV_PACKAGE_SUGGESTS "perl >= 5.0, perl-File-Which, perl-File-Listing, perl-URI")
+set(CPACK_RPM_DEV_PACKAGE_REQUIRES "file, ${HIP_DEV_ROCM_PKG_DEPENDENCIES}, hsa-rocr-devel, hipcc")
+
+set(CPACK_RPM_DEV_PACKAGE_PROVIDES "hip-base")
+set(CPACK_RPM_DEV_PACKAGE_OBSOLETES "hip-base")
+#End dev Packaging setting
+
+#Begin doc Packaging setting
+set(CPACK_DOC_DEB "ON")
+set(CPACK_DOC_RPM "ON")
+set(CPACK_DEBIAN_DOC_PACKAGE_NAME "hip-doc")
+set(CPACK_RPM_DOC_PACKAGE_NAME "hip-doc")
+set(CPACK_COMPONENT_DOC_DESCRIPTION "HIP: Heterogenous-computing Interface for Portability [DOCUMENTATION]")
+
+set(CPACK_DEBIAN_DOC_PACKAGE_DEPENDS "hip-dev (= ${CPACK_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}), rocm-core")
+set(CPACK_DEBIAN_DOC_PACKAGE_PROVIDES "hip-doc")
+
+string(REPLACE "-" "_" HIP_BASE_VERSION ${CPACK_PACKAGE_VERSION})
+set(CPACK_RPM_DOC_PACKAGE_REQUIRES "hip-devel = ${HIP_BASE_VERSION}-${CPACK_RPM_PACKAGE_RELEASE}, rocm-core")
+
+#End doc Packaging setting
+
+#Begin runtime-nvidia Packaging setting
+set(CPACK_RUNTIME-NVIDIA_DEB "ON")
+set(CPACK_RUNTIME-NVIDIA_RPM "ON")
+set(CPACK_DEBIAN_RUNTIME-NVIDIA_PACKAGE_NAME "hip-runtime-nvidia")
+set(CPACK_RPM_RUNTIME-NVIDIA_PACKAGE_NAME "hip-runtime-nvidia")
+set(CPACK_COMPONENT_RUNTIME-NVIDIA_DESCRIPTION "HIP: Heterogenous-computing Interface for Portability [RUNTIME-NVIDIA]")
+
+set(CPACK_DEBIAN_RUNTIME-NVIDIA_PACKAGE_DEPENDS "cuda (>= 7.5), rocm-core, hipcc-nvidia")
+set(CPACK_DEBIAN_RUNTIME-NVIDIA_PACKAGE_PROVIDES "hip-nvcc")
+set(CPACK_DEBIAN_RUNTIME-NVIDIA_PACKAGE_REPLACES "hip-nvcc")
+
+set(CPACK_RPM_RUNTIME-NVIDIA_PACKAGE_PROVIDES "hip-nvcc")
+set(CPACK_RPM_RUNTIME-NVIDIA_PACKAGE_OBSOLETES "hip-nvcc")
+set(CPACK_RPM_RUNTIME-NVIDIA_PACKAGE_REQUIRES "cuda >= 7.5, rocm-core, hipcc-nvidia")
+
+# Begin asan Packaging setting
+set(CPACK_ASAN_DEB "ON")
+set(CPACK_ASAN_RPM "ON")
+set(CPACK_DEBIAN_ASAN_PACKAGE_NAME "hip-runtime-amd-asan")
+set(CPACK_RPM_ASAN_PACKAGE_NAME "hip-runtime-amd-asan")
+set(CPACK_COMPONENT_ASAN_DESCRIPTION "HIP:Heterogenous-computing Interface for Portability [AddressSanitizer libraries]")
+set(HIP_ASAN_ROCM_PKG_DEPENDENCIES "hsa-rocr-asan, rocminfo, comgr-asan, rocm-llvm, rocm-core-asan")
+set(CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS "${HIP_ASAN_ROCM_PKG_DEPENDENCIES}, ${HIP_RUNTIME_DEB_STDPKG_DEPENDENCIES}")
+set(CPACK_RPM_ASAN_PACKAGE_REQUIRES "${HIP_ASAN_ROCM_PKG_DEPENDENCIES}, ${HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES}")
+#End asan Packaging setting
+
+# Remove dependency on rocm-core if -DROCM_DEP_ROCMCORE=ON not given to cmake
+if(NOT ROCM_DEP_ROCMCORE)
+
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_BINARY_PACKAGE_REQUIRES ${CPACK_RPM_BINARY_PACKAGE_REQUIRES})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS ${CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_DEV_PACKAGE_REQUIRES ${CPACK_RPM_DEV_PACKAGE_REQUIRES})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_DEV_PACKAGE_DEPENDS ${CPACK_DEBIAN_DEV_PACKAGE_DEPENDS})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_DOC_PACKAGE_REQUIRES ${CPACK_RPM_DOC_PACKAGE_REQUIRES})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_DOC_PACKAGE_DEPENDS ${CPACK_DEBIAN_DOC_PACKAGE_DEPENDS})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_RUNTIME-NVIDIA_PACKAGE_REQUIRES ${CPACK_RPM_RUNTIME-NVIDIA_PACKAGE_REQUIRES})
+    string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_RUNTIME-NVIDIA_PACKAGE_DEPENDS ${CPACK_DEBIAN_RUNTIME-NVIDIA_PACKAGE_DEPENDS})
+    string(REGEX REPLACE ",? ?rocm-core-asan" "" CPACK_RPM_ASAN_PACKAGE_REQUIRES ${CPACK_RPM_ASAN_PACKAGE_REQUIRES})
+    string(REGEX REPLACE ",? ?rocm-core-asan" "" CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS})
+endif()
+# package name and dependencies for static package
+if(NOT BUILD_SHARED_LIBS)
+   set(CPACK_RPM_STATIC_PACKAGE_NAME "hip-static-devel")
+   set(CPACK_DEBIAN_STATIC_PACKAGE_NAME "hip-static-dev")
+   set(CPACK_COMPONENT_STATIC_DESCRIPTION "HIP:Heterogenous-computing Interface for Portability [Static Libraries - AMD]")
+
+   set(CPACK_RPM_STATIC_PACKAGE_REQUIRES "${CPACK_RPM_DEV_PACKAGE_REQUIRES}, ${HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES}")
+   string(REGEX REPLACE ",? ?hsa-rocr-devel" "" CPACK_RPM_STATIC_PACKAGE_REQUIRES ${CPACK_RPM_STATIC_PACKAGE_REQUIRES})
+   string(REGEX REPLACE ",? ?hipcc" "" CPACK_RPM_STATIC_PACKAGE_REQUIRES ${CPACK_RPM_STATIC_PACKAGE_REQUIRES})
+   string(REGEX REPLACE ",? ?hip-runtime-amd" "" CPACK_RPM_STATIC_PACKAGE_REQUIRES ${CPACK_RPM_STATIC_PACKAGE_REQUIRES})
+   string(APPEND CPACK_RPM_STATIC_PACKAGE_REQUIRES  ", hsa-rocr-static-devel, hipcc-static-devel, rocminfo-static-devel, comgr-static-devel")
+
+   set(CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS "${CPACK_DEBIAN_DEV_PACKAGE_DEPENDS}, ${HIP_RUNTIME_RPM_STDPKG_DEPENDENCIES}")
+   string(REGEX REPLACE ",? ?hsa-rocr-dev" "" CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS ${CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS})
+   string(REGEX REPLACE ",? ?hipcc" "" CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS ${CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS})
+   string(REGEX REPLACE ",? ?hip-runtime-amd" "" CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS ${CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS})
+   string(APPEND CPACK_DEBIAN_STATIC_PACKAGE_DEPENDS  ", hsa-rocr-static-dev, hipcc-static-dev, rocminfo-static-dev, comgr-static-dev")
+endif()
+include(CPack)
+#static package generation
+# Group binary and dev component to single package
+if(NOT BUILD_SHARED_LIBS)
+    cpack_add_component_group("static")
+    cpack_add_component( binary  GROUP static )
+    cpack_add_component( dev GROUP static )
+endif()

projects/clr/hipamd/packaging/convert_md_to_html.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# Copyright (c) 2016 - 2021 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+function die {
+    echo "${1-Died}." >&2
+    exit 1
+}
+
+function cleanup {
+    rm -rf "$workdir"
+}
+
+# parse arguments
+hip_srcdir=$1
+html_destdir=$2
+[ "$hip_srcdir" != "" ] || [ "$html_destdir" != "" ] || die "Invalid arguments!"
+
+# create temporary directory for grip settings
+workdir=`mktemp -d`
+trap cleanup EXIT
+
+# setup grip
+export GRIPURL=$hip_srcdir
+export GRIPHOME=$workdir
+echo "CACHE_DIRECTORY = '$html_destdir/asset'" > $workdir/settings.py
+mkdir -p $html_destdir $html_destdir/docs/markdown
+
+# convert all md files to html
+pushd $hip_srcdir
+for f in *.md docs/markdown/*.md; do grip --export --no-inline $f $html_destdir/${f%.*}.html; done
+popd
+
+# convert absolute links to relative links
+pushd $html_destdir
+for f in *.html; do sed -i "s?$GRIPURL/??g" $f; done
+for f in docs/markdown/*.html; do sed -i "s?$GRIPURL/?../../?g" $f; done
+popd
+
+# update document titles
+pushd $html_destdir
+for f in *.html; do sed -i "s?.md - Grip??g" $f; done
+for f in docs/markdown/*.html; do sed -i "s?.md - Grip??g" $f; done
+popd
+
+# replace .md with .html in links
+pushd $html_destdir
+for f in *.html; do sed -i "s?.md\"?.html\"?g" $f; done
+for f in *.html; do sed -i "s?.md#?.html#?g" $f; done
+for f in docs/markdown/*.html; do sed -i "s?.md\"?.html\"?g" $f; done
+for f in docs/markdown/*.html; do sed -i "s?.md#?.html#?g" $f; done
+popd
+
+# replace github.io links
+pushd $html_destdir
+sed -i "s?http://rocm-developer-tools.github.io/HIP?docs/RuntimeAPI/html/index.html?g" README.html
+sed -i "s?http://rocm-developer-tools.github.io/HIP?docs/RuntimeAPI/html/?g" RELEASE.html
+popd
+
+exit 0

projects/clr/hipamd/packaging/hip-devel.postinst

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) 2016 - 2021 Advanced Micro Devices, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+ROCMDIR=@ROCM_PATH@
+HIPINCDIR=$ROCMDIR/@CMAKE_INSTALL_INCLUDEDIR@/hip
+CURRENTDIR=`pwd`
+
+#FILE_REORG_BACKWARD_COMPATIBILITY
+HIPINCDIR=$ROCMDIR/hip/include/hip

projects/clr/hipamd/packaging/hip-devel.prerm

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright (c) 2016 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+ROCMDIR=@ROCM_PATH@
+CURRENTDIR=`pwd`
+
+HIPINCDIR=$ROCMDIR/@CMAKE_INSTALL_INCLUDEDIR@/hip
+([ ! -d $HIPINCDIR ]) && exit 0
+
+#FILE_REORG_BACKWARD_COMPATIBILITY
+  #backward copatibility code , to be removed later
+HIPDIR=$ROCMDIR/hip
+HIPINCDIR=$ROCMDIR/hip/include/hip
+([ ! -d $HIPINCDIR ]) && exit 0
+([ ! -d $HIPDIR ]) && exit 0
+rmdir --ignore-fail-on-non-empty $HIPDIR

projects/clr/hipamd/packaging/hip-runtime-amd.postinst

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright (c) 2020 - 2022 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+ROCMDIR=@ROCM_PATH@
+ROCMCMAKEDIR=$ROCMDIR/@CMAKE_INSTALL_LIBDIR@/cmake
+HIPCMAKEDIR=$ROCMDIR/hip/lib/cmake
+CURRENTDIR=`pwd`
+
+mkdir -p $HIPCMAKEDIR/hip
+mkdir -p $HIPCMAKEDIR/hip-lang
+mkdir -p $HIPCMAKEDIR/hiprtc
+
+HIPTARGETFILES=$(ls -A $ROCMCMAKEDIR/hip | grep  "^hip-targets")
+cd $HIPCMAKEDIR/hip
+for f in $HIPTARGETFILES
+do
+    ln -s -r -f $ROCMCMAKEDIR/hip/$f $(basename $f)
+done
+cd $CURRENTDIR
+
+HIPLANGTARGETFILES=$(ls -A $ROCMCMAKEDIR/hip-lang | grep  "^hip-lang-targets")
+cd $HIPCMAKEDIR/hip-lang
+for f in $HIPLANGTARGETFILES
+do
+    ln -s -r -f $ROCMCMAKEDIR/hip-lang/$f $(basename $f)
+done
+cd $CURRENTDIR
+
+HIPRTCTARGETFILES=$(ls -A $ROCMCMAKEDIR/hiprtc | grep  "^hiprtc-targets")
+cd $HIPCMAKEDIR/hiprtc
+for f in $HIPRTCTARGETFILES
+do
+    ln -s -r -f $ROCMCMAKEDIR/hiprtc/$f $(basename $f)
+done
+cd $CURRENTDIR

projects/clr/hipamd/packaging/hip-runtime-amd.prerm

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Copyright (c) 2016 - 2022 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+ROCMDIR=@ROCM_PATH@
+HIPDIR=$ROCMDIR/hip
+HIPCMAKEDIR=$ROCMDIR/hip/lib/cmake/hip
+HIPLANGCMAKEDIR=$ROCMDIR/hip/lib/cmake/hip-lang
+HIPRTCCMAKEDIR=$ROCMDIR/hip/lib/cmake/hiprtc
+CURRENTDIR=`pwd`
+([ ! -d $ROCMDIR ] || [ ! -d $HIPDIR ]) && exit 0
+
+([ ! -d $HIPCMAKEDIR ] ) && exit 0
+# Remove soft-links to hip-target
+HIPTARGETFILES=$(ls -A $HIPCMAKEDIR | grep "^hip-targets")
+
+cd  $HIPCMAKEDIR
+for f in $HIPTARGETFILES; do
+    [ -e $f ] || continue
+    rm $(basename $f)
+done
+cd $CURRENTDIR
+([ ! -d $HIPLANGCMAKEDIR ] ) && exit 0
+# Remove soft-links to hip-lang-target
+HIPLANGTARGETFILES=$(ls -A $HIPLANGCMAKEDIR | grep "^hip-lang-targets")
+
+cd  $HIPLANGCMAKEDIR
+for f in $HIPLANGTARGETFILES; do
+    [ -e $f ] || continue
+    rm $(basename $f)
+done
+
+cd $CURRENTDIR
+
+([ ! -d $HIPRTCCMAKEDIR ] ) && exit 0
+# Remove soft-links to hiprtc-target
+HIPRTCTARGETFILES=$(ls -A $HIPRTCCMAKEDIR | grep "^hiprtc-targets")
+
+cd  $HIPRTCCMAKEDIR
+for f in $HIPRTCTARGETFILES; do
+    [ -e $f ] || continue
+    rm $(basename $f)
+done
+
+cd $CURRENTDIR
+
+rmdir --ignore-fail-on-non-empty $HIPCMAKEDIR
+rmdir --ignore-fail-on-non-empty $HIPLANGCMAKEDIR
+rmdir --ignore-fail-on-non-empty $HIPRTCCMAKEDIR

projects/clr/hipamd/packaging/hip-runtime-nvidia.postinst

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2020 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+ROCMDIR=@ROCM_PATH@
+HIPDIR=$ROCMDIR/hip
+
+if [ -d $ROCMDIR ] ; then
+    ln -s -f $ROCMDIR /opt/rocm
+fi

projects/clr/hipamd/packaging/hip-runtime-nvidia.prerm

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright (c) 2020 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+if [ -L "/opt/rocm" ] ; then
+   unlink /opt/rocm
+fi

projects/clr/hipamd/src/CMakeLists.txt

@@ -0,0 +1,357 @@
+# Copyright (c) 2020 - 2022 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+include(GNUInstallDirs)
+
+set(VERSION_MAJOR_AMDHIP ${HIP_VERSION_MAJOR})
+set(VERSION_MINOR_AMDHIP ${HIP_VERSION_MINOR})
+
+if(ADDRESS_SANITIZER)
+  set(ASAN_LINKER_FLAGS "-fsanitize=address")
+  set(ASAN_COMPILER_FLAGS "-fno-omit-frame-pointer -fsanitize=address")
+
+  if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    if(BUILD_SHARED_LIBS)
+      set(ASAN_COMPILER_FLAGS "${ASAN_COMPILER_FLAGS} -shared-libsan")
+      set(ASAN_LINKER_FLAGS "${ASAN_LINKER_FLAGS} -shared-libsan")
+    else()
+      set(ASAN_LINKER_FLAGS "${ASAN_LINKER_FLAGS} -static-libsan")
+    endif()
+  endif()
+
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ASAN_COMPILER_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ASAN_COMPILER_FLAGS}")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_LINKER_FLAGS} -s -Wl,--build-id=sha1")
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${ASAN_LINKER_FLAGS} -Wl,--build-id=sha1")
+endif()
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=deprecated-declarations")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations")
+endif()
+
+option(DISABLE_DIRECT_DISPATCH "Disable Direct Dispatch" OFF)
+
+option(BUILD_SHARED_LIBS "Build the shared library" ON)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
+
+if(BUILD_SHARED_LIBS)
+  add_library(amdhip64 SHARED)
+  if(WIN32)
+    set_target_properties(amdhip64 PROPERTIES RUNTIME_OUTPUT_NAME "amdhip64_${HIP_VERSION_MAJOR}")
+  endif()
+  # Windows doesn't have a strip utility, so CMAKE_STRIP won't be set.
+  if((CMAKE_BUILD_TYPE STREQUAL "Release") AND NOT ("${CMAKE_STRIP}" STREQUAL ""))
+    add_custom_command(TARGET amdhip64 POST_BUILD COMMAND ${CMAKE_STRIP} $<TARGET_FILE:amdhip64>)
+  endif()
+else()
+  add_library(amdhip64 STATIC $<TARGET_OBJECTS:rocclr>)
+endif()
+
+set_target_properties(amdhip64 PROPERTIES
+  CXX_STANDARD 17
+  CXX_STANDARD_REQUIRED ON
+  CXX_EXTENSIONS OFF
+  POSITION_INDEPENDENT_CODE ON
+  # Workaround for many places in the HIP project
+  # having hardcoded references to build/lib/libamdhip64.so
+  LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib
+  ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
+
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+  set_target_properties(amdhip64 PROPERTIES OUTPUT_NAME "amdhip64")
+else()
+  set_target_properties(amdhip64 PROPERTIES OUTPUT_NAME "amdhip32")
+endif()
+
+# Disable versioning for Windows
+# as currently HIP_LIB_VERSION_STRING and HIP_LIB_VERSION_MAJOR
+# are not being populated
+if(NOT WIN32)
+  if(BUILD_SHARED_LIBS)
+    set_target_properties(amdhip64 PROPERTIES
+      VERSION ${HIP_LIB_VERSION_STRING}
+      SOVERSION ${HIP_LIB_VERSION_MAJOR})
+  endif()
+endif()
+
+target_sources(amdhip64 PRIVATE
+  fixme.cpp
+  hip_activity.cpp
+  hip_code_object.cpp
+  hip_context.cpp
+  hip_device_runtime.cpp
+  hip_device.cpp
+  hip_error.cpp
+  hip_event.cpp
+  hip_event_ipc.cpp
+  hip_fatbin.cpp
+  hip_global.cpp
+  hip_graph_internal.cpp
+  hip_graph.cpp
+  hip_hmm.cpp
+  hip_intercept.cpp
+  hip_memory.cpp
+  hip_mempool.cpp
+  hip_mempool_impl.cpp
+  hip_module.cpp
+  hip_peer.cpp
+  hip_platform.cpp
+  hip_profile.cpp
+  hip_stream_ops.cpp
+  hip_stream.cpp
+  hip_surface.cpp
+  hip_texture.cpp
+  hip_gl.cpp
+  hip_vm.cpp
+  hip_api_trace.cpp
+  hip_table_interface.cpp
+  hip_table_interface_c.cpp
+  hip_comgr_helper.cpp)
+
+if(WIN32)
+  target_sources(amdhip64 PRIVATE hip_runtime.cpp)
+endif()
+
+if(BUILD_SHARED_LIBS)
+  if(WIN32)
+    target_sources(amdhip64 PRIVATE amdhip.def)
+  else()
+    target_link_libraries(amdhip64 PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/hip_hcc.map.in")
+    set_target_properties(amdhip64 PROPERTIES LINK_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/hip_hcc.map.in")
+  endif()
+endif()
+
+if(WIN32)
+  configure_file(hip_hcc_in.rc.in hip_hcc_info.rc @ONLY)
+  target_sources(amdhip64 PRIVATE hip_hcc_info.rc)
+endif()
+
+target_include_directories(amdhip64
+  PRIVATE
+    ${HIP_COMMON_INCLUDE_DIR}
+    ${PROJECT_SOURCE_DIR}/include
+    ${PROJECT_BINARY_DIR}/include)
+
+target_compile_definitions(amdhip64 PRIVATE __HIP_PLATFORM_AMD__)
+target_link_libraries(amdhip64 PRIVATE ${OPENGL_LIBRARIES})
+target_link_libraries(amdhip64 PRIVATE ${CMAKE_DL_LIBS})
+# Add link to comgr, hsa-runtime and other required libraries in target files
+# This is required for static libraries
+if(NOT BUILD_SHARED_LIBS)
+  find_package(hsa-runtime64)
+  find_package(amd_comgr)
+  target_link_libraries(amdhip64 PRIVATE
+    pthread numa rt c amd_comgr hsa-runtime64::hsa-runtime64)
+endif()
+
+# Note in static case we cannot link against rocclr.
+# If we would, we'd also have to export rocclr and have hipcc pass it to the linker.
+if(BUILD_SHARED_LIBS)
+  target_link_libraries(amdhip64 PRIVATE rocclr)
+else()
+  target_compile_definitions(amdhip64 PRIVATE $<TARGET_PROPERTY:rocclr,COMPILE_DEFINITIONS>)
+  target_include_directories(amdhip64 PRIVATE $<TARGET_PROPERTY:rocclr,INCLUDE_DIRECTORIES>)
+endif()
+
+if(DISABLE_DIRECT_DISPATCH)
+  target_compile_definitions(amdhip64 PRIVATE DISABLE_DIRECT_DISPATCH)
+endif()
+
+# hipamd will reference llvm symbols, so we need install rocm-llvm-dev package
+find_package(LLVM REQUIRED CONFIG
+  HINTS
+    ${ROCM_PATH}/llvm)
+message(STATUS "Found LLVM_INCLUDE_DIRS=" ${LLVM_INCLUDE_DIRS} )
+target_include_directories(amdhip64 PRIVATE ${LLVM_INCLUDE_DIRS})
+
+# Short-Term solution for pre-compiled headers for online compilation
+# Enable pre compiled header
+if(__HIP_ENABLE_PCH)
+  # find_package(LLVM) returns the lib/cmake/llvm location. We require the root.
+  if(NOT DEFINED HIP_LLVM_ROOT)
+    set(HIP_LLVM_ROOT "${LLVM_DIR}/../../..")
+  endif()
+  execute_process(COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/hip_embed_pch.sh ${HIP_COMMON_INCLUDE_DIR} ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${HIP_LLVM_ROOT}" COMMAND_ECHO STDERR RESULT_VARIABLE EMBED_PCH_RC WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+  if (EMBED_PCH_RC AND NOT EMBED_PCH_RC EQUAL 0)
+    message(FATAL_ERROR "Failed to embed PCH")
+  endif()
+
+  target_compile_definitions(amdhip64 PRIVATE __HIP_ENABLE_PCH)
+  target_sources(amdhip64 PRIVATE ${CMAKE_BINARY_DIR}/hip_pch.o)
+endif()
+
+# Add hiprtc
+add_subdirectory(hiprtc)
+
+#############################
+# Profiling API support
+#############################
+# Generate profiling API macros/structures header
+option(USE_PROF_API "Enable roctracer integration" ON)
+# Enable profiling API
+if(USE_PROF_API)
+  set(PROF_API_STR "${PROJECT_BINARY_DIR}/include/hip/amd_detail/hip_prof_str.h")
+  set(PROF_API_STR_IN "${CMAKE_SOURCE_DIR}/hipamd/include/hip/amd_detail/hip_prof_str.h")
+  set(PROF_API_HDR "${HIP_COMMON_INCLUDE_DIR}/hip/hip_runtime_api.h")
+  set(PROF_GL_HDR "${CMAKE_SOURCE_DIR}/hipamd/include/hip/amd_detail/amd_hip_gl_interop.h")
+  set(PROF_API_DEPRECATED "${HIP_COMMON_INCLUDE_DIR}/hip/hip_deprecated.h")
+  set(PROF_API_SRC "${CMAKE_CURRENT_SOURCE_DIR}")
+  set(PROF_API_GEN "${CMAKE_CURRENT_SOURCE_DIR}/hip_prof_gen.py")
+  set(PROF_API_LOG "${PROJECT_BINARY_DIR}/hip_prof_gen.log.txt")
+  set(PROF_API_NEWHDR "${PROJECT_BINARY_DIR}/new_header.h")
+  find_package(Python3 COMPONENTS Interpreter REQUIRED)
+
+  execute_process(COMMAND ${Python3_EXECUTABLE} -c "import CppHeaderParser"
+                  RESULT_VARIABLE CPP_HEADER_PARSER
+                  OUTPUT_QUIET)
+
+  if(NOT ${CPP_HEADER_PARSER} EQUAL 0)
+    message(FATAL_ERROR "\
+    The \"CppHeaderParser\" Python3 package is not installed. \
+    Please install it using the following command: \"pip3 install CppHeaderParser\".\
+  ")
+  endif()
+
+  add_custom_command(OUTPUT ${PROF_API_NEWHDR}.i
+    COMMAND ${CMAKE_COMMAND} -E cat ${PROF_API_HDR} ${PROF_GL_HDR} > ${PROF_API_NEWHDR}
+    COMMAND ${CMAKE_C_COMPILER}
+        "-D$<JOIN:$<TARGET_PROPERTY:amdhip64,COMPILE_DEFINITIONS>,;-D>"
+        "-I$<JOIN:$<TARGET_PROPERTY:amdhip64,INCLUDE_DIRECTORIES>,;-I>"
+        "-DHIP_INCLUDE_HIP_HIP_RUNTIME_PT_API_H=1"
+        ${c_flags}
+        $<TARGET_PROPERTY:amdhip64,COMPILE_OPTIONS>
+        ${CPP_EXTRA_C_FLAGS}
+        -E ${PROF_API_NEWHDR} -o ${PROF_API_NEWHDR}.i
+    COMMAND_EXPAND_LISTS VERBATIM
+    IMPLICIT_DEPENDS C ${PROF_API_HDR} ${PROF_GL_HDR} ${PROF_API_DEPRECATED}
+    DEPENDS ${PROF_API_HDR} ${PROF_GL_HDR} ${PROF_API_DEPRECATED}
+    COMMENT "Generating new header from hip_runtime_api.h")
+
+  add_custom_command(OUTPUT ${PROF_API_STR}
+    COMMAND ${Python3_EXECUTABLE} ${PROF_API_GEN} -v -t --priv ${PROF_API_NEWHDR}.i ${PROF_API_SRC} ${PROF_API_STR_IN} ${PROF_API_STR}
+    DEPENDS ${PROF_API_STR_IN} ${PROF_API_NEWHDR}.i ${PROF_API_GEN}
+    COMMENT "Generating profiling primitives: ${PROF_API_STR}")
+
+  add_custom_target(gen-prof-api-str-header ALL
+    DEPENDS ${PROF_API_STR}
+    SOURCES ${PROF_API_NEWHDR}.i)
+
+  set_target_properties(amdhip64 PROPERTIES PUBLIC_HEADER ${PROF_API_STR})
+
+  find_path(PROF_API_HEADER_DIR prof_protocol.h
+    HINTS
+    ${PROF_API_HEADER_PATH}
+    PATHS
+    ${ROCM_PATH}/roctracer
+    PATH_SUFFIXES
+    include/ext)
+
+  if(NOT PROF_API_HEADER_DIR)
+    message(WARNING "Profiling API header not found. Disabling roctracer integration. Use -DPROF_API_HEADER_PATH=<path to prof_protocol.h header>")
+  else()
+    target_include_directories(amdhip64 PUBLIC ${PROF_API_HEADER_DIR})
+    message(STATUS "Profiling API: ${PROF_API_HEADER_DIR}")
+  endif()
+
+  add_dependencies(amdhip64 gen-prof-api-str-header)
+endif()
+
+target_compile_definitions(amdhip64 PUBLIC USE_PROF_API=1)
+
+if(WIN32 OR NOT BUILD_SHARED_LIBS)
+    # rocprofiler-register is not support on Windows
+    set(HIP_ENABLE_ROCPROFILER_REGISTER OFF)
+else()
+  option(HIP_ENABLE_ROCPROFILER_REGISTER "Enable rocprofiler-register support" ON)
+endif()
+
+if(HIP_ENABLE_ROCPROFILER_REGISTER)
+    find_package(rocprofiler-register REQUIRED
+        HINTS $ENV{rocprofiler_register_ROOT} $ENV{ROCPROFILER_REGISTER_ROOT} ${CMAKE_INSTALL_PREFIX}
+        PATHS /opt/rocm)
+
+    # don't use HIP_VERSION_PATCH because it is too large (> 100) for rocprofiler register
+    target_compile_definitions(amdhip64 PRIVATE HIP_ROCPROFILER_REGISTER=1
+        HIP_ROCP_REG_VERSION_MAJOR=${HIP_VERSION_MAJOR}
+        HIP_ROCP_REG_VERSION_MINOR=${HIP_VERSION_MINOR}
+        HIP_ROCP_REG_VERSION_PATCH=0)
+    target_link_libraries(amdhip64 PRIVATE rocprofiler-register::rocprofiler-register)
+    set_target_properties(amdhip64 PROPERTIES INSTALL_RPATH "\$ORIGIN")
+endif()
+
+add_custom_command(TARGET amdhip64 POST_BUILD COMMAND
+  ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/share/hip/.hipInfo ${PROJECT_BINARY_DIR}/lib/.hipInfo)
+add_custom_command(TARGET amdhip64 POST_BUILD COMMAND
+  ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include)
+add_custom_command(TARGET amdhip64 POST_BUILD COMMAND
+  ${CMAKE_COMMAND} -E copy_directory ${HIP_COMMON_INCLUDE_DIR} ${PROJECT_BINARY_DIR}/include)
+
+add_library(host INTERFACE)
+target_link_libraries(host INTERFACE amdhip64)
+
+add_library(device INTERFACE)
+target_link_libraries(device INTERFACE host)
+
+# Current packaging assumes that HIP runtime will always be installed in ${ROCM_PATH}/lib
+# This is false to assume, because some distros like CentOS will use the lib64 directory instead of lib
+# Relying on CMake to choose the library directory for us will default in that case to lib64
+# Hence there will be a mismatch between where HIP is installed and where CMake thinks it is
+
+INSTALL(TARGETS amdhip64 host device
+  EXPORT hip-targets
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+INSTALL(EXPORT hip-targets DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip NAMESPACE hip::)
+
+INSTALL(TARGETS amdhip64 host device
+  EXPORT hip-lang-targets
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+INSTALL(EXPORT hip-lang-targets DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hip-lang NAMESPACE hip-lang::)
+
+include(CMakePackageConfigHelpers)
+
+configure_package_config_file(
+  ${HIP_COMMON_DIR}/hip-lang-config.cmake.in
+  ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config.cmake
+  INSTALL_DESTINATION ${CONFIG_LANG_PACKAGE_INSTALL_DIR}
+  NO_SET_AND_CHECK_MACRO
+  NO_CHECK_REQUIRED_COMPONENTS_MACRO
+  PATH_VARS LIB_INSTALL_DIR INCLUDE_INSTALL_DIR BIN_INSTALL_DIR)
+
+write_basic_package_version_file(
+  ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config-version.cmake
+  VERSION "${HIP_VERSION_MAJOR}.${HIP_VERSION_MINOR}.${HIP_VERSION_PATCH}"
+  COMPATIBILITY SameMajorVersion)
+install(
+    FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config-version.cmake
+    DESTINATION
+    ${CONFIG_LANG_PACKAGE_INSTALL_DIR}/
+    )

projects/clr/hipamd/src/amdhip.def

@@ -0,0 +1,494 @@
+EXPORTS
+hipChooseDevice
+hipChooseDeviceR0000
+hipChooseDeviceR0600
+hipCtxCreate
+hipCtxDestroy
+hipCtxDisablePeerAccess
+hipCtxEnablePeerAccess
+hipCtxGetApiVersion
+hipCtxGetCacheConfig
+hipCtxGetCurrent
+hipCtxGetDevice
+hipCtxGetFlags
+hipCtxGetSharedMemConfig
+hipCtxPopCurrent
+hipCtxPushCurrent
+hipCtxSetCacheConfig
+hipCtxSetCurrent
+hipCtxSetSharedMemConfig
+hipCtxSynchronize
+hipDeviceCanAccessPeer
+hipDeviceComputeCapability
+hipDeviceDisablePeerAccess
+hipDeviceEnablePeerAccess
+hipDeviceGet
+hipDeviceGetAttribute
+hipDeviceGetByPCIBusId
+hipDeviceGetCacheConfig
+hipDeviceGetStreamPriorityRange
+hipDeviceGetLimit
+hipDeviceGetName
+hipDeviceGetTexture1DLinearMaxWidth
+hipDeviceGetUuid
+hipDeviceGetPCIBusId
+hipDeviceGetSharedMemConfig
+hipDeviceGetP2PAttribute
+hipExternalMemoryGetMappedMipmappedArray
+hipDevicePrimaryCtxGetState
+hipDevicePrimaryCtxRelease
+hipDevicePrimaryCtxReset
+hipDevicePrimaryCtxRetain
+hipDevicePrimaryCtxSetFlags
+hipDeviceReset
+hipDeviceSetCacheConfig
+hipDeviceSetSharedMemConfig
+hipDeviceSynchronize
+hipDeviceTotalMem
+hipDriverGetVersion
+hipEventCreate
+hipEventCreateWithFlags
+hipEventDestroy
+hipEventElapsedTime
+hipEventQuery
+hipEventRecord
+hipEventSynchronize
+hipExtGetLinkTypeAndHopCount
+hipExtLaunchMultiKernelMultiDevice
+hipExtMallocWithFlags
+hipExtModuleLaunchKernel
+hipExtLaunchKernel
+hipFree
+hipFreeArray
+hipFuncSetAttribute
+hipFuncSetCacheConfig
+hipFuncSetSharedMemConfig
+hipGetDevice
+hipGetDeviceCount
+hipGetDeviceProperties
+hipGetDevicePropertiesR0000
+hipGetDevicePropertiesR0600
+hipGetErrorName
+hipGetErrorString
+hipGetLastError
+hipMemAllocHost
+hipHostAlloc
+hipHostFree
+hipHostGetDevicePointer
+hipHostGetFlags
+hipHostMalloc
+hipHostRegister
+hipHostUnregister
+hipInit
+hipIpcCloseMemHandle
+hipIpcGetMemHandle
+hipIpcOpenMemHandle
+hipIpcGetEventHandle
+hipIpcOpenEventHandle
+hipMalloc
+hipMalloc3D
+hipMalloc3DArray
+hipMallocManaged
+hipDeviceGetDefaultMemPool
+hipDeviceSetMemPool
+hipDeviceGetMemPool
+hipMallocAsync
+hipFreeAsync
+hipMemPoolTrimTo
+hipMemPoolSetAttribute
+hipMemPoolGetAttribute
+hipMemPoolSetAccess
+hipMemPoolGetAccess
+hipMemPoolCreate
+hipMemPoolDestroy
+hipMallocFromPoolAsync
+hipMemPoolExportToShareableHandle
+hipMemPoolImportFromShareableHandle
+hipMemPoolExportPointer
+hipMemPoolImportPointer
+hipArrayCreate
+hipArray3DCreate
+hipArrayDestroy
+hipArrayGetInfo
+hipArrayGetDescriptor
+hipArray3DGetDescriptor
+hipMallocArray
+hipMemAdvise
+hipMemAllocPitch
+hipMallocPitch
+hipMemcpy
+hipMemcpyWithStream
+hipMemcpyParam2D
+hipMemcpy2D
+hipMemcpy2DAsync
+hipMemcpy2DToArray
+hipMemcpy2DToArrayAsync
+hipMemcpy3D
+hipMemcpy3DAsync
+hipDrvMemcpy3D
+hipDrvMemcpy3DAsync
+hipMemcpyAsync
+hipMemcpyDtoD
+hipMemcpyDtoDAsync
+hipMemcpyDtoH
+hipMemcpyDtoHAsync
+hipMemcpyFromSymbol
+hipMemcpyFromSymbolAsync
+hipMemcpyHtoD
+hipMemcpyHtoDAsync
+hipMemcpyPeer
+hipMemcpyPeerAsync
+hipMemcpyToArray
+hipMemcpyFromArray
+hipMemcpyToSymbol
+hipMemcpyToSymbolAsync
+hipMemGetAddressRange
+hipGetSymbolAddress
+hipGetSymbolSize
+hipMemGetInfo
+hipMemPrefetchAsync
+hipMemPtrGetInfo
+hipMemRangeGetAttribute
+hipMemRangeGetAttributes
+hipMemset
+hipMemsetAsync
+hipMemsetD8
+hipMemsetD8Async
+hipMemsetD16
+hipMemsetD16Async
+hipMemsetD32
+hipMemsetD32Async
+hipMemset2D
+hipMemset2DAsync
+hipMemset3D
+hipMemset3DAsync
+hipModuleGetFunction
+hipModuleGetGlobal
+hipModuleGetTexRef
+hipModuleLaunchKernel
+hipModuleLaunchCooperativeKernel
+hipModuleLaunchCooperativeKernelMultiDevice
+hipLaunchCooperativeKernel
+hipLaunchCooperativeKernelMultiDevice
+hipHccModuleLaunchKernel
+hipModuleLoad
+hipModuleLoadData
+hipModuleLoadDataEx
+hipModuleUnload
+hipModuleOccupancyMaxPotentialBlockSize
+hipModuleOccupancyMaxPotentialBlockSizeWithFlags
+hipModuleOccupancyMaxActiveBlocksPerMultiprocessor
+hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags
+hipOccupancyMaxPotentialBlockSize
+hipOccupancyMaxActiveBlocksPerMultiprocessor
+hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags
+hipFuncGetAttribute
+hipFuncGetAttributes
+hipPeekAtLastError
+hipPointerGetAttributes
+hipProfilerStart
+hipProfilerStop
+hipRuntimeGetVersion
+hipGetDeviceFlags
+hipSetDevice
+hipSetDeviceFlags
+hipStreamAddCallback
+hipStreamAttachMemAsync
+hipStreamCreate
+hipStreamCreateWithFlags
+hipStreamCreateWithPriority
+hipStreamDestroy
+hipStreamGetDevice
+hipStreamGetFlags
+hipStreamQuery
+hipStreamSynchronize
+hipStreamWaitEvent
+__hipPopCallConfiguration
+__hipPushCallConfiguration
+__hipRegisterFatBinary
+__hipRegisterFunction
+__hipRegisterVar
+__hipRegisterSurface
+__hipRegisterTexture
+__hipRegisterManagedVar
+__hipUnregisterFatBinary
+hipConfigureCall
+hipSetupArgument
+hipLaunchByPtr
+hipLaunchKernel
+hipRegisterTracerCallback
+hipApiName
+hipKernelNameRef
+hipBindTexture
+hipBindTexture2D
+hipBindTextureToArray
+hipBindTextureToMipmappedArray
+hipGetTextureAlignmentOffset
+hipGetTextureReference
+hipUnbindTexture
+hipCreateChannelDesc
+hipCreateTextureObject
+hipDestroyTextureObject
+hipGetChannelDesc
+hipGetTextureObjectResourceDesc
+hipGetTextureObjectResourceViewDesc
+hipGetTextureObjectTextureDesc
+hipTexRefGetAddress
+hipTexRefGetAddressMode
+hipTexRefGetArray
+hipTexRefGetBorderColor
+hipTexRefGetFilterMode
+hipTexRefGetFlags
+hipTexRefGetFormat
+hipTexRefGetMaxAnisotropy
+hipTexRefGetMipmapFilterMode
+hipTexRefGetMipmapLevelBias
+hipTexRefGetMipmapLevelClamp
+hipTexRefGetMipMappedArray
+hipTexRefSetAddress
+hipTexRefSetAddress2D
+hipTexRefSetAddressMode
+hipTexRefSetArray
+hipTexRefSetBorderColor
+hipTexRefSetFilterMode
+hipTexRefSetFlags
+hipTexRefSetFormat
+hipTexRefSetMaxAnisotropy
+hipTexRefSetMipmapFilterMode
+hipTexRefSetMipmapLevelBias
+hipTexRefSetMipmapLevelClamp
+hipTexRefSetMipmappedArray
+hipProfilerStart
+hipProfilerStop
+hipCreateSurfaceObject
+hipDestroySurfaceObject
+hipGetCmdName
+hipMipmappedArrayCreate
+hipMallocMipmappedArray
+hipMipmappedArrayDestroy
+hipFreeMipmappedArray
+hipMipmappedArrayGetLevel
+hipGetMipmappedArrayLevel
+hipMallocHost
+hipFreeHost
+hipTexObjectCreate
+hipTexObjectDestroy
+hipTexObjectGetResourceDesc
+hipTexObjectGetResourceViewDesc
+hipTexObjectGetTextureDesc
+hipExtStreamCreateWithCUMask
+hipStreamGetPriority
+hipMemcpy2DFromArray
+hipMemcpy2DFromArrayAsync
+hipDrvMemcpy2DUnaligned
+hipMemcpyAtoH
+hipMemcpyHtoA
+hipMemcpyParam2DAsync
+__gnu_h2f_ieee
+__gnu_f2h_ieee
+hipExtStreamGetCUMask
+hipImportExternalMemory
+hipExternalMemoryGetMappedBuffer
+hipDestroyExternalMemory
+hipGraphCreate
+hipGraphDestroy
+hipGraphAddKernelNode
+hipGraphAddMemsetNode
+hipGraphAddMemcpyNode
+hipGraphAddMemcpyNode1D
+hipGraphInstantiate
+hipGraphLaunch
+hipStreamIsCapturing
+hipStreamBeginCapture
+hipStreamEndCapture
+hipGraphExecDestroy
+hipPointerGetAttribute
+hipDrvPointerGetAttributes
+hipImportExternalSemaphore
+hipSignalExternalSemaphoresAsync
+hipWaitExternalSemaphoresAsync
+hipDestroyExternalSemaphore
+hipGLGetDevices
+hipGraphicsGLRegisterBuffer
+hipGraphicsGLRegisterImage
+hipGraphicsMapResources
+hipGraphicsResourceGetMappedPointer
+hipGraphicsSubResourceGetMappedArray
+hipGraphicsUnmapResources
+hipGraphicsUnregisterResource
+hipGraphGetNodes
+hipGraphGetRootNodes
+hipGraphKernelNodeGetParams
+hipGraphKernelNodeSetParams
+hipGraphKernelNodeSetAttribute
+hipGraphKernelNodeGetAttribute
+hipGraphMemcpyNodeGetParams
+hipGraphMemcpyNodeSetParams
+hipGraphMemsetNodeGetParams
+hipGraphMemsetNodeSetParams
+hipGraphAddDependencies
+hipGraphExecKernelNodeSetParams
+hipGraphAddEmptyNode
+hipStreamGetCaptureInfo
+hipStreamGetCaptureInfo_v2
+hipStreamUpdateCaptureDependencies
+hipGraphRemoveDependencies
+hipGraphGetEdges
+hipGraphNodeGetDependencies
+hipGraphNodeGetDependentNodes
+hipGraphNodeGetType
+hipGraphDestroyNode
+hipGraphClone
+hipGraphNodeFindInClone
+hipGraphAddChildGraphNode
+hipGraphChildGraphNodeGetGraph
+hipGraphExecChildGraphNodeSetParams
+hipGraphAddMemcpyNodeFromSymbol
+hipGraphMemcpyNodeSetParamsFromSymbol
+hipGraphExecMemcpyNodeSetParamsFromSymbol
+hipGraphAddMemcpyNodeToSymbol
+hipGraphMemcpyNodeSetParamsToSymbol
+hipGraphExecMemcpyNodeSetParamsToSymbol
+hipGraphExecMemcpyNodeSetParams
+hipGraphMemcpyNodeSetParams1D
+hipGraphExecMemcpyNodeSetParams1D
+hipGraphAddEventRecordNode
+hipGraphEventRecordNodeGetEvent
+hipGraphEventRecordNodeSetEvent
+hipGraphExecEventRecordNodeSetEvent
+hipGraphAddEventWaitNode
+hipGraphEventWaitNodeGetEvent
+hipGraphEventWaitNodeSetEvent
+hipGraphExecEventWaitNodeSetEvent
+hipGraphAddHostNode
+hipGraphHostNodeGetParams
+hipGraphHostNodeSetParams
+hipGraphExecHostNodeSetParams
+hipGraphExecUpdate
+hipGraphInstantiateWithFlags
+hipGraphExecMemsetNodeSetParams
+hipDeviceGetGraphMemAttribute
+hipDeviceSetGraphMemAttribute
+hipDeviceGraphMemTrim
+amd_dbgapi_get_build_name
+amd_dbgapi_get_git_hash
+amd_dbgapi_get_build_id
+hipThreadExchangeStreamCaptureMode
+hipMemAddressFree
+hipMemAddressReserve
+hipMemCreate
+hipMemExportToShareableHandle
+hipMemGetAccess
+hipMemGetAllocationGranularity
+hipMemGetAllocationPropertiesFromHandle
+hipMemImportFromShareableHandle
+hipMemMap
+hipMemMapArrayAsync
+hipMemRelease
+hipMemRetainAllocationHandle
+hipMemSetAccess
+hipMemUnmap
+hipMemcpy_spt
+hipMemcpyAsync_spt
+hipStreamSynchronize_spt
+hipMemcpyToSymbol_spt
+hipMemcpyFromSymbol_spt
+hipMemcpy2D_spt
+hipMemcpy2DToArray_spt
+hipMemcpy2DFromArray_spt
+hipMemcpy3D_spt
+hipMemset_spt
+hipMemset2D_spt
+hipMemset3D_spt
+hipStreamQuery_spt
+hipStreamGetFlags_spt
+hipStreamGetPriority_spt
+hipStreamWaitEvent_spt
+hipEventRecord_spt
+hipLaunchKernel_spt
+hipLaunchCooperativeKernel_spt
+hipStreamWriteValue32
+hipStreamWriteValue64
+hipStreamWaitValue32
+hipStreamWaitValue64
+hipDeviceSetLimit
+hipGetStreamDeviceId
+hipGraphLaunch_spt
+hipStreamBeginCapture_spt
+hipStreamEndCapture_spt
+hipStreamIsCapturing_spt
+hipStreamGetCaptureInfo_spt
+hipStreamGetCaptureInfo_v2_spt
+hipStreamAddCallback_spt
+hipMemsetAsync_spt
+hipMemset2DAsync_spt
+hipMemset3DAsync_spt
+hipMemcpy3DAsync_spt
+hipMemcpy2DAsync_spt
+hipMemcpyFromSymbolAsync_spt
+hipMemcpyToSymbolAsync_spt
+hipMemcpyFromArray_spt
+hipMemcpy2DFromArrayAsync_spt
+hipMemcpy2DToArrayAsync_spt
+hipDrvGetErrorName
+hipDrvGetErrorString
+hipUserObjectCreate
+hipUserObjectRelease
+hipUserObjectRetain
+hipGraphRetainUserObject
+hipGraphReleaseUserObject
+hipLaunchHostFunc
+hipLaunchHostFunc_spt
+hipGraphDebugDotPrint
+hipGraphKernelNodeCopyAttributes
+hipGraphNodeGetEnabled
+hipGraphNodeSetEnabled
+hipGraphUpload
+hipGraphAddMemAllocNode
+hipGraphMemAllocNodeGetParams
+hipGraphAddMemFreeNode
+hipGraphMemFreeNodeGetParams
+hipDrvGraphAddMemcpyNode
+hipDrvGraphAddMemsetNode
+hipGetProcAddress
+hipExtGetLastError
+hipGraphAddExternalSemaphoresSignalNode
+hipGraphAddExternalSemaphoresWaitNode
+hipGraphExternalSemaphoresSignalNodeSetParams
+hipGraphExternalSemaphoresSignalNodeGetParams
+hipGraphExternalSemaphoresWaitNodeSetParams
+hipGraphExternalSemaphoresWaitNodeGetParams
+hipGraphExecExternalSemaphoresSignalNodeSetParams
+hipGraphExecExternalSemaphoresWaitNodeSetParams
+hipGraphAddNode
+hipGraphInstantiateWithParams
+hipStreamBeginCaptureToGraph
+hipGetFuncBySymbol
+hipDrvGraphAddMemFreeNode
+hipDrvGraphExecMemcpyNodeSetParams
+hipDrvGraphExecMemsetNodeSetParams
+hipSetValidDevices
+hipMemcpyAtoD
+hipMemcpyDtoA
+hipMemcpyAtoA
+hipMemcpyAtoHAsync
+hipMemcpyHtoAAsync
+hipMemcpy2DArrayToArray
+hipGraphExecGetFlags
+hipGraphNodeSetParams
+hipGraphExecNodeSetParams
+hipDrvGraphMemcpyNodeSetParams
+hipDrvGraphMemcpyNodeGetParams
+hipStreamBatchMemOp
+hipGraphAddBatchMemOpNode
+hipGraphBatchMemOpNodeGetParams
+hipGraphBatchMemOpNodeSetParams
+hipGraphExecBatchMemOpNodeSetParams
+hipEventRecordWithFlags
+hipLinkAddData
+hipLinkAddFile
+hipLinkComplete
+hipLinkCreate
+hipLinkDestroy
+hipLaunchKernelExC
+hipDrvLaunchKernelEx

projects/clr/hipamd/src/cmake/FindROCclr.cmake

@@ -0,0 +1,51 @@
+# Copyright (c) 2020 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+if(ROCCLR_FOUND)
+  return()
+endif()
+
+find_path(ROCCLR_INCLUDE_DIR top.hpp
+  HINTS
+    ${ROCCLR_PATH}
+  PATHS
+    # gerrit repo name
+    ${CMAKE_SOURCE_DIR}/vdi
+    ${CMAKE_SOURCE_DIR}/../vdi
+    ${CMAKE_SOURCE_DIR}/../../vdi
+    # github repo name
+    ${CMAKE_SOURCE_DIR}/ROCclr
+    ${CMAKE_SOURCE_DIR}/../ROCclr
+    ${CMAKE_SOURCE_DIR}/../../ROCclr
+    # jenkins repo name
+    ${CMAKE_SOURCE_DIR}/rocclr
+    ${CMAKE_SOURCE_DIR}/../rocclr
+    ${CMAKE_SOURCE_DIR}/../../rocclr
+  PATH_SUFFIXES
+    include)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(ROCclr
+  "\nROCclr not found"
+  ROCCLR_INCLUDE_DIR)
+mark_as_advanced(ROCCLR_INCLUDE_DIR)
+
+list(APPEND CMAKE_MODULE_PATH "${ROCCLR_INCLUDE_DIR}/../cmake")
+include(ROCclr)

projects/clr/hipamd/src/fixme.cpp

@@ -0,0 +1,33 @@
+/* Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+#include "vdi_common.hpp"
+#ifdef _WIN32
+#include <windows.h>
+#include <d3d9.h>
+#include <d3d10_1.h>
+#include <CL/cl_d3d10.h>
+#include <CL/cl_d3d11.h>
+#include <CL/cl_dx9_media_sharing.h>
+#endif
+#include <CL/cl_icd.h>
+
+cl_icd_dispatch amd::ICDDispatchedObject::icdVendorDispatch_[] = {0};
+amd::PlatformIDS amd::PlatformID::Platform = {amd::ICDDispatchedObject::icdVendorDispatch_};

projects/clr/hipamd/src/hip_activity.cpp

@@ -0,0 +1,26 @@
+/* Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+#include "platform/activity.hpp"
+#include <hip/hip_runtime_api.h>
+
+extern "C" const char* hipGetCmdName(unsigned op) {
+  return amd::activity_prof::getOclCommandKindString(static_cast<cl_command_type>(op));
+}