Update run-ci.py (#641)

* Temp: Fixing node id * source formatting (clang-format v11) (#709) Co-authored-by: ammarwa <3832908+ammarwa@users.noreply.github.com> * Using logical node id * Update agent.cpp * Update agent.cpp * Python formatting * Update run-ci.py * Update run-ci.py * Update continuous_integration.yml * Update continuous_integration.yml running directly using the prepared runner container * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update run-ci.py * Clean up * Fixing install paths * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Update continuous_integration.yml * Fixing GPU Agents Test Validation * python formatting (black) (#712) Co-authored-by: ammarwa <3832908+ammarwa@users.noreply.github.com> * Fixing the issue with rocclr detected kernels __amd_rocclr_.* * python formatting (black) (#713) Co-authored-by: ammarwa <3832908+ammarwa@users.noreply.github.com> * Fixing the issue with rocclr detected kernels __amd_rocclr_.* * Fixing static number of async copies and using hsa_api instead for validation * python formatting (black) (#714) Co-authored-by: ammarwa <3832908+ammarwa@users.noreply.github.com> * Increasing the time limit for waiting on active signals * Update continuous_integration.yml * Update async_copy.cpp * Update CMakeLists.txt * changing node id to logical node id in rocprofv3 * Update tool.cpp * testing async mem copy signal decrement * Update logging.cpp * Update validate.py --------- Co-authored-by: Ammar ELWazir <aelwazir@rocprofiler1.amd.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: ammarwa <3832908+ammarwa@users.noreply.github.com> Co-authored-by: Ammar ELWazir <aelwazir@rocprofiler2.amd.com> Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
2024-04-02 01:39:24 -05:00
@@ -11,6 +11,8 @@ on:
    paths-ignore:
      - '*.md'
      - 'source/docs/**'
+  issue_comment:
+    types: [created]

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -28,11 +30,32 @@ jobs:
    strategy:
      fail-fast: true
      matrix:
-        os: ['ubuntu-22.04']
-        runner: ['vega20']
-        build-type: ['RelWithDebInfo']
+        runner: [mi200-runner-set, mi300-runner-set, vega20-runner-set, navi3-runner-set, rocprofiler-runner-set-multi-gpus]
+        vega20:
+          - ${{ github.event_name ==  'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'vega20') }}
+        navi3:
+          - ${{ github.event_name ==  'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'navi3') }}
+        mi300:
+          - ${{ github.event_name ==  'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'mi300') }}
+        mi200:
+          - ${{ github.event_name ==  'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'mi200') }}
+        general:
+          - ${{ github.event_name ==  'pull_request' || github.event_name ==  'push' }}
+        os: [ubuntu-22.04]
+        build-type: [RelWithDebInfo]
        ci-flags: ['--linter clang-tidy']
        name-tag: ['']
+        exclude:
+          - mi300: false
+            runner: mi300-runner-set
+          - mi200: false
+            runner: mi200-runner-set
+          - navi3: false
+            runner: navi3-runner-set
+          - vega20: false
+            runner: vega20-runner-set
+          - general: false
+            runner: rocprofiler-runner-set-multi-gpus

    runs-on: ${{ matrix.runner }}

@@ -40,18 +63,8 @@ jobs:
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1

-    container:
-      # TODO(aelwazir): Adding multiple os distros
-      image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest
-      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
-      credentials:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
    steps:
    - uses: actions/checkout@v4
-      with:
-        submodules: true

    - name: Install requirements
      shell: bash
@@ -72,15 +85,16 @@ jobs:
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python python3 git cmake ctest; do which-realpath $i; done
        ls -la
+        cat /sys/class/kfd/kfd/topology/nodes/*/gpu_id || true; /opt/rocm/bin/rocminfo | grep gfx

    - name: Configure, Build, and Test
      timeout-minutes: 30
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}${{ matrix.name-tag }}
-          --build-jobs 8
-          --site ${{ matrix.runner }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}${{ matrix.name-tag }}
+          --build-jobs 16
+          --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p')
          --gpu-targets ${{ env.GPU_LIST }}
          ${{ matrix.ci-flags }}
          --
@@ -95,12 +109,12 @@ jobs:
    - name: Install
      timeout-minutes: 10
      run:
-        cmake --build build --target install --parallel 8
+        cmake --build build --target install --parallel 16

    - name: Build Packaging
      timeout-minutes: 10
      run:
-        cmake --build build --target package --parallel 8
+        cmake --build build --target package --parallel 16

    - name: Test Install Build
      timeout-minutes: 10
@@ -109,8 +123,8 @@ jobs:
        CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-samples samples
        CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-tests tests
        export LD_LIBRARY_PATH=/opt/rocprofiler-sdk/lib:${LD_LIBRARY_PATH}
-        cmake --build build-samples --target all --parallel 8
-        cmake --build build-tests --target all --parallel 8
+        cmake --build build-samples --target all --parallel 16
+        cmake --build build-tests --target all --parallel 16
        ctest --test-dir build-samples --output-on-failure
        ctest --test-dir build-tests --output-on-failure

@@ -129,8 +143,8 @@ jobs:
      run: |
        CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-samples-deb /opt/rocm/share/rocprofiler-sdk/samples
        CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-tests-deb /opt/rocm/share/rocprofiler-sdk/tests
-        cmake --build build-samples-deb --target all --parallel 8
-        cmake --build build-tests-deb --target all --parallel 8
+        cmake --build build-samples-deb --target all --parallel 16
+        cmake --build build-tests-deb --target all --parallel 16
        ctest --test-dir build-samples-deb --output-on-failure
        ctest --test-dir build-tests-deb --output-on-failure

@@ -147,41 +161,18 @@ jobs:
    strategy:
      fail-fast: true
      matrix:
-      # TODO: Change it back when re-enabling on mi200
        include:
            - os: 'ubuntu-22.04'
-              runner: 'vega20'
+              runner: 'rocprofiler-cluster-runner-set'
              build-type: 'Release'
-        # include:
-        #     - os: 'ubuntu-22.04'
-        #       runner: 'renderD131'
-        #       device: '/renderD131'
-        #       build-type: 'Release'

+    if: ${{ contains(github.event_name, 'pull_request') }}
    runs-on: ${{ matrix.runner }}

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1

-    # TODO: Uncomment this when re-enabling tests on the mi200 as it contains --memory and --cpus flag for the mi200. Remove these 2 options when running on vega20.
-    # vega20 machine only has 24 cpus available.
-    container:
-      # TODO(aelwazir): Adding multiple os distros
-      image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest
-      options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
-      credentials:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    # container:
-    #   # TODO(aelwazir): Adding multiple os distros
-    #   image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest
-    #   options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --device=/dev/dri${{ matrix.device }} --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
-    #   credentials:
-    #     username: ${{ secrets.DOCKER_USERNAME }}
-    #     password: ${{ secrets.DOCKER_PASSWORD }}
-
    steps:
    - name: Patch Git
      timeout-minutes: 25
@@ -191,10 +182,9 @@ jobs:
        add-apt-repository -y ppa:git-core/ppa
        apt-get update
        apt-get install -y git
+        cat /sys/class/kfd/kfd/topology/nodes/*/gpu_id || true; /opt/rocm/bin/rocminfo | grep gfx

    - uses: actions/checkout@v4
-      with:
-        submodules: true

    - name: Load Existing XML Code Coverage
      if: github.event_name == 'pull_request'
@@ -239,9 +229,9 @@ jobs:
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-codecov
-          --build-jobs 8
-          --site ${{ matrix.runner }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-codecov
+          --build-jobs 16
+          --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p')
          --gpu-targets ${{ env.GPU_LIST }}
          --coverage all
          --
@@ -254,9 +244,9 @@ jobs:
      run:
        find build -type f | egrep '\.gcda$' | xargs rm &&
        python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-codecov-tests
-          --build-jobs 8
-          --site ${{ matrix.runner }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-codecov-tests
+          --build-jobs 16
+          --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p')
          --gpu-targets ${{ env.GPU_LIST }}
          --coverage tests
          --
@@ -269,9 +259,9 @@ jobs:
      run:
        find build -type f | egrep '\.gcda$' | xargs rm &&
        python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-codecov-samples
-          --build-jobs 8
-          --site ${{ matrix.runner }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-codecov-samples
+          --build-jobs 16
+          --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p')
          --gpu-targets ${{ env.GPU_LIST }}
          --coverage samples
          --
@@ -379,29 +369,20 @@ jobs:
      fail-fast: false
      matrix:
        os: ['ubuntu-22.04']
-        runner: ['vega20']
+        runner: ['rocprofiler-cluster-runner-set']
        build-type: ['RelWithDebInfo']
        sanitizer: ['AddressSanitizer', 'ThreadSanitizer', 'LeakSanitizer']
        ci-flags: ['']

+    if: ${{ contains(github.event_name, 'pull_request') }}
    runs-on: ${{ matrix.runner }}

    # define this for containers
    env:
      GIT_DISCOVERY_ACROSS_FILESYSTEM: 1

-    container:
-      # TODO(aelwazir): Adding multiple os distros
-      image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest
-      options: --privileged --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined
-      credentials:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
    steps:
    - uses: actions/checkout@v4
-      with:
-        submodules: true

    - name: List Files
      shell: bash
@@ -409,6 +390,7 @@ jobs:
        which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; }
        for i in python python3 git cmake ctest; do which-realpath $i; done
        ls -la
+        cat /sys/class/kfd/kfd/topology/nodes/*/gpu_id || true; /opt/rocm/bin/rocminfo | grep gfx

    - name: Install requirements
      shell: bash
@@ -432,9 +414,9 @@ jobs:
      shell: bash
      run:
        python3 ./source/scripts/run-ci.py -B build
-          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-${{ matrix.sanitizer }}
-          --build-jobs 8
-          --site ${{ matrix.runner }}
+          --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.sanitizer }}
+          --build-jobs 16
+          --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p')
          --gpu-targets ${{ env.GPU_LIST }}
          --memcheck=${{ matrix.sanitizer }}
          ${{ matrix.ci-flags }}
@@ -116,8 +116,6 @@ update_logging(const logging_config& cfg, bool setup_env, int env_override)
    if(setup_env)
    {
        common::set_env("GOOGLE_LOG_DIR", get_env("PWD", ""), env_override);
-        common::set_env("GOOGLE_LOGTOSTDERR", cfg.loglevel, env_override);
-        common::set_env("GOOGLE_ALSOLOGTOSTDERR", cfg.alsologtostderr, env_override);
    }
 }
 }  // namespace common
@@ -660,10 +660,12 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
                    record);

                auto kernel_trace_ss = std::stringstream{};
+                // TODO(aelwazir): To be changed back to use node id once ROCR fixes
+                // the hsa_agents to use the real node id
                tool::csv::kernel_trace_csv_encoder::write_row(
                    kernel_trace_ss,
                    CHECK_NOTNULL(buffered_name_info)->kind_names.at(record->kind),
-                    agent_info->at(record->agent_id)->node_id,
+                    agent_info->at(record->agent_id)->logical_node_id,
                    record->queue_id.handle,
                    record->kernel_id,
                    std::move(kernel_name),
@@ -710,14 +712,16 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
                    static_cast<rocprofiler_buffer_tracing_memory_copy_record_t*>(header->payload);

                auto memory_copy_trace_ss = std::stringstream{};
+                // TODO(aelwazir): To be changed back to use node id once ROCR fixes
+                // the hsa_agents to use the real node id
                tool::csv::memory_copy_csv_encoder::write_row(
                    memory_copy_trace_ss,
                    CHECK_NOTNULL(buffered_name_info)->kind_names.at(record->kind),
                    CHECK_NOTNULL(buffered_name_info)
                        ->operation_names.at(record->kind)
                        .at(record->operation),
-                    agent_info->at(record->src_agent_id)->node_id,
-                    agent_info->at(record->dst_agent_id)->node_id,
+                    agent_info->at(record->src_agent_id)->logical_node_id,
+                    agent_info->at(record->dst_agent_id)->logical_node_id,
                    record->correlation_id.internal,
                    record->start_timestamp,
                    record->end_timestamp);
@@ -930,10 +934,12 @@ counter_record_callback(rocprofiler_profile_counting_dispatch_data_t dispatch_da
        using csv_encoder = tool::csv::counter_collection_csv_encoder;

        auto counter_collection_ss = std::stringstream{};
+        // TODO(aelwazir): To be changed back to use node id once ROCR fixes
+        // the hsa_agents to use the real node id
        csv_encoder::write_row(counter_collection_ss,
                               correlation_id.internal,
                               cnt_dispatch_data_v->dispatch_index,
-                               agent_info->at(dispatch_data.agent_id)->node_id,
+                               agent_info->at(dispatch_data.agent_id)->logical_node_id,
                               dispatch_data.queue_id.handle,
                               getpid(),
                               cnt_dispatch_data_v->thread_id,
@@ -963,7 +969,9 @@ list_metrics_iterate_agents(rocprofiler_agent_version_t,
    {
        const auto* agent      = static_cast<const rocprofiler_agent_v0_t*>(agents[idx]);
        auto        counters_v = counter_vec_t{};
-        uint32_t    node_id    = agent->node_id;
+        // TODO(aelwazir): To be changed back to use node id once ROCR fixes
+        // the hsa_agents to use the real node id
+        uint32_t node_id = agent->logical_node_id;
        ROCPROFILER_CALL(
            rocprofiler_iterate_agent_supported_counters(
                agent->id,
@@ -272,9 +272,10 @@ async_copy_handler(hsa_signal_value_t signal_value, void* arg)
    {
        ROCP_HSA_TABLE_CALL(ERROR, get_core_table()->hsa_signal_destroy_fn(_data->rocp_signal));
        delete _data;
-        get_active_signals()->fetch_sub(1);
    }

+    get_active_signals()->fetch_sub(1);
+
    if(_corr_id) _corr_id->sub_ref_count();

    return (signal_value > 0);
@@ -12,8 +12,8 @@ import multiprocessing

 # this constant is used to define CTEST_PROJECT_NAME
 # and default value for CTEST_SUBMIT_URL
-_PROJECT_NAME = "rocprofiler-v2-internal"
-_BASE_URL = "10.194.116.31/cdash"
+_PROJECT_NAME = "rocprofiler-sdk-internal"
+_BASE_URL = "cdash.mariadb.svc.cluster.local"
 _GCOVR_GENERATE_CMD = None


@@ -42,6 +42,15 @@ def test_counter_values(input_data):
                f"Debug Info:\n {str(data)}"
            )

+    for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"][
+        "counter_collection"
+    ]:
+        value = itr["counter_value"]
+        assert int(round(value, 0)) == int(round(1 * scaling_factor, 0)), (
+            f"Failure on agent "
+            f"{str(itr)} expected {1 * scaling_factor} but got {value} "
+            f"Debug Info:\n {str(data)}"
+        )

 if __name__ == "__main__":
    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
@@ -4,6 +4,7 @@ import sys
 import pytest
 import numpy as np
 import pandas as pd
+import re

 kernel_list = sorted(
    ["addition_kernel", "subtract_kernel", "multiply_kernel", "divide_kernel"]
@@ -23,11 +24,19 @@ def test_validate_counter_collection_pmc1(input_data: pd.DataFrame):
    assert (df["Process_Id"].astype(int).values > 0).all()
    assert len(df["Kernel_Name"]) > 0

-    assert kernel_list == sorted(df["Kernel_Name"].unique().tolist())
+    counter_collection_pmc1_kernel_list = [
+        x
+        for x in sorted(df["Kernel_Name"].unique().tolist())
+        if not re.search(r"__amd_rocclr_.*", x)
+    ]
+
+    assert kernel_list == counter_collection_pmc1_kernel_list

    kernel_count = dict([[itr, 0] for itr in kernel_list])
    assert len(kernel_count) == len(kernel_list)
    for itr in df["Kernel_Name"]:
+        if re.search(r"__amd_rocclr_.*", itr):
+            continue
        kernel_count[itr] += 1
    kn_cnt = [itr for _, itr in kernel_count.items()]
    assert min(kn_cnt) == max(kn_cnt) and len(unique(kn_cnt)) == 1
@@ -27,7 +27,7 @@ set_tests_properties(
    rocprofv3-test-trace-hip-in-libraries-execute
    PROPERTIES
        TIMEOUT
-        45
+        100
        LABELS
        "integration-tests"
        ENVIRONMENT
@@ -1,7 +1,9 @@
 #!/usr/bin/env python3

+import re
 import sys
 import pytest
+import subprocess


 class dim3(object):
@@ -79,7 +81,6 @@ def test_api_trace(hsa_input_data, hip_input_data):
 def test_kernel_trace(kernel_input_data):
    valid_kernel_names = sorted(
        [
-            "__amd_rocclr_fillBufferAligned",
            "(anonymous namespace)::transpose(int const*, int*, int, int)",
            "void (anonymous namespace)::addition_kernel<float>(float*, float const*, float const*, int, int)",
            "void (anonymous namespace)::divide_kernel<float>(float*, float const*, float const*, int, int)",
@@ -91,6 +92,8 @@ def test_kernel_trace(kernel_input_data):
    kernels = []
    for row in kernel_input_data:
        kernel_name = row["Kernel_Name"]
+        if re.search(r"__amd_rocclr_.*", kernel_name):
+            continue

        assert row["Kind"] == "KERNEL_DISPATCH"
        assert int(row["Agent_Id"]) > 0
@@ -123,18 +126,34 @@ def test_kernel_trace(kernel_input_data):
    assert kernels == valid_kernel_names


-def test_memory_copy_trace(memory_copy_input_data):
+def test_memory_copy_trace(memory_copy_input_data, hsa_input_data):
    for row in memory_copy_input_data:
        assert row["Kind"] == "MEMORY_COPY"
        assert row["Direction"] in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
        if row["Direction"] == "HOST_TO_DEVICE":
-            assert int(row["Source_Agent_Id"]) == 0
+            output = subprocess.check_output(
+                'rocminfo | grep "Node: *'
+                + row["Source_Agent_Id"]
+                + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
+                shell=True,
+            )
+            assert int(str(output).find("CPU")) >= 0
        elif row["Direction"] == "DEVICE_TO_HOST":
-            assert int(row["Destination_Agent_Id"]) == 0
+            output = subprocess.check_output(
+                'rocminfo | grep "Node: *'
+                + row["Destination_Agent_Id"]
+                + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
+                shell=True,
+            )
+            assert int(str(output).find("CPU")) >= 0
        assert int(row["Correlation_Id"]) > 0
        assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])

-    assert len(memory_copy_input_data) == 120
+    valid_length = 0
+    for row in hsa_input_data:
+        if re.search(r".*memory_async_copy.*", row["Function"]):
+            valid_length += 1
+    assert len(memory_copy_input_data) == valid_length


 if __name__ == "__main__":
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3

 import sys
+import subprocess
 import pytest


@@ -67,15 +68,39 @@ def test_memory_copy_trace(memory_copy_input_data):

    row = memory_copy_input_data[0]
    assert row["Direction"] == "HOST_TO_DEVICE"
-    assert int(row["Source_Agent_Id"]) == 0
-    assert int(row["Destination_Agent_Id"]) >= 1
+    output = subprocess.check_output(
+        'rocminfo | grep "Node: *'
+        + row["Source_Agent_Id"]
+        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
+        shell=True,
+    )
+    assert int(str(output).find("CPU")) >= 0
+    output = subprocess.check_output(
+        'rocminfo | grep "Node: *'
+        + row["Destination_Agent_Id"]
+        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
+        shell=True,
+    )
+    assert int(str(output).find("GPU")) >= 0
    assert int(row["Correlation_Id"]) > 0
    assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])

    row = memory_copy_input_data[1]
    assert row["Direction"] == "DEVICE_TO_HOST"
-    assert int(row["Source_Agent_Id"]) >= 1
-    assert int(row["Destination_Agent_Id"]) == 0
+    output = subprocess.check_output(
+        'rocminfo | grep "Node: *'
+        + row["Source_Agent_Id"]
+        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
+        shell=True,
+    )
+    assert int(str(output).find("GPU")) >= 0
+    output = subprocess.check_output(
+        'rocminfo | grep "Node: *'
+        + row["Destination_Agent_Id"]
+        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
+        shell=True,
+    )
+    assert int(str(output).find("CPU")) >= 0
    assert int(row["Correlation_Id"]) > 0
    assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])