diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 598e02054e..a2df0e6ed9 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -11,6 +11,8 @@ on: paths-ignore: - '*.md' - 'source/docs/**' + issue_comment: + types: [created] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -28,11 +30,32 @@ jobs: strategy: fail-fast: true matrix: - os: ['ubuntu-22.04'] - runner: ['vega20'] - build-type: ['RelWithDebInfo'] + runner: [mi200-runner-set, mi300-runner-set, vega20-runner-set, navi3-runner-set, rocprofiler-runner-set-multi-gpus] + vega20: + - ${{ github.event_name == 'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'vega20') }} + navi3: + - ${{ github.event_name == 'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'navi3') }} + mi300: + - ${{ github.event_name == 'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'mi300') }} + mi200: + - ${{ github.event_name == 'issue_comment' && contains(github.event.comment.body, 'rerun') && contains(github.event.comment.body, 'mi200') }} + general: + - ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + os: [ubuntu-22.04] + build-type: [RelWithDebInfo] ci-flags: ['--linter clang-tidy'] name-tag: [''] + exclude: + - mi300: false + runner: mi300-runner-set + - mi200: false + runner: mi200-runner-set + - navi3: false + runner: navi3-runner-set + - vega20: false + runner: vega20-runner-set + - general: false + runner: rocprofiler-runner-set-multi-gpus runs-on: ${{ matrix.runner }} @@ -40,18 +63,8 @@ jobs: env: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 - container: - # TODO(aelwazir): Adding multiple os distros - image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest - options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined - credentials: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - steps: - uses: actions/checkout@v4 - with: - submodules: true - name: Install requirements shell: bash @@ -72,15 +85,16 @@ jobs: which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; } for i in python python3 git cmake ctest; do which-realpath $i; done ls -la + cat /sys/class/kfd/kfd/topology/nodes/*/gpu_id || true; /opt/rocm/bin/rocminfo | grep gfx - name: Configure, Build, and Test timeout-minutes: 30 shell: bash run: python3 ./source/scripts/run-ci.py -B build - --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}${{ matrix.name-tag }} - --build-jobs 8 - --site ${{ matrix.runner }} + --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}${{ matrix.name-tag }} + --build-jobs 16 + --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p') --gpu-targets ${{ env.GPU_LIST }} ${{ matrix.ci-flags }} -- @@ -95,12 +109,12 @@ jobs: - name: Install timeout-minutes: 10 run: - cmake --build build --target install --parallel 8 + cmake --build build --target install --parallel 16 - name: Build Packaging timeout-minutes: 10 run: - cmake --build build --target package --parallel 8 + cmake --build build --target package --parallel 16 - name: Test Install Build timeout-minutes: 10 @@ -109,8 +123,8 @@ jobs: CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-samples samples CMAKE_PREFIX_PATH=/opt/rocprofiler-sdk cmake -B build-tests tests export LD_LIBRARY_PATH=/opt/rocprofiler-sdk/lib:${LD_LIBRARY_PATH} - cmake --build build-samples --target all --parallel 8 - cmake --build build-tests --target all --parallel 8 + cmake --build build-samples --target all --parallel 16 + cmake --build build-tests --target all --parallel 16 ctest --test-dir build-samples --output-on-failure ctest --test-dir build-tests --output-on-failure @@ -129,8 +143,8 @@ jobs: run: | CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-samples-deb /opt/rocm/share/rocprofiler-sdk/samples CMAKE_PREFIX_PATH=/opt/rocm cmake -B build-tests-deb /opt/rocm/share/rocprofiler-sdk/tests - cmake --build build-samples-deb --target all --parallel 8 - cmake --build build-tests-deb --target all --parallel 8 + cmake --build build-samples-deb --target all --parallel 16 + cmake --build build-tests-deb --target all --parallel 16 ctest --test-dir build-samples-deb --output-on-failure ctest --test-dir build-tests-deb --output-on-failure @@ -147,41 +161,18 @@ jobs: strategy: fail-fast: true matrix: - # TODO: Change it back when re-enabling on mi200 include: - os: 'ubuntu-22.04' - runner: 'vega20' + runner: 'rocprofiler-cluster-runner-set' build-type: 'Release' - # include: - # - os: 'ubuntu-22.04' - # runner: 'renderD131' - # device: '/renderD131' - # build-type: 'Release' + if: ${{ contains(github.event_name, 'pull_request') }} runs-on: ${{ matrix.runner }} # define this for containers env: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 - # TODO: Uncomment this when re-enabling tests on the mi200 as it contains --memory and --cpus flag for the mi200. Remove these 2 options when running on vega20. - # vega20 machine only has 24 cpus available. - container: - # TODO(aelwazir): Adding multiple os distros - image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest - options: --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined - credentials: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # container: - # # TODO(aelwazir): Adding multiple os distros - # image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest - # options: --memory=128g --cpus=32 --ipc=host --device=/dev/kfd --device=/dev/dri${{ matrix.device }} --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined - # credentials: - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - steps: - name: Patch Git timeout-minutes: 25 @@ -191,10 +182,9 @@ jobs: add-apt-repository -y ppa:git-core/ppa apt-get update apt-get install -y git + cat /sys/class/kfd/kfd/topology/nodes/*/gpu_id || true; /opt/rocm/bin/rocminfo | grep gfx - uses: actions/checkout@v4 - with: - submodules: true - name: Load Existing XML Code Coverage if: github.event_name == 'pull_request' @@ -239,9 +229,9 @@ jobs: shell: bash run: python3 ./source/scripts/run-ci.py -B build - --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-codecov - --build-jobs 8 - --site ${{ matrix.runner }} + --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-codecov + --build-jobs 16 + --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p') --gpu-targets ${{ env.GPU_LIST }} --coverage all -- @@ -254,9 +244,9 @@ jobs: run: find build -type f | egrep '\.gcda$' | xargs rm && python3 ./source/scripts/run-ci.py -B build - --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-codecov-tests - --build-jobs 8 - --site ${{ matrix.runner }} + --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-codecov-tests + --build-jobs 16 + --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p') --gpu-targets ${{ env.GPU_LIST }} --coverage tests -- @@ -269,9 +259,9 @@ jobs: run: find build -type f | egrep '\.gcda$' | xargs rm && python3 ./source/scripts/run-ci.py -B build - --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-codecov-samples - --build-jobs 8 - --site ${{ matrix.runner }} + --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-codecov-samples + --build-jobs 16 + --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p') --gpu-targets ${{ env.GPU_LIST }} --coverage samples -- @@ -379,29 +369,20 @@ jobs: fail-fast: false matrix: os: ['ubuntu-22.04'] - runner: ['vega20'] + runner: ['rocprofiler-cluster-runner-set'] build-type: ['RelWithDebInfo'] sanitizer: ['AddressSanitizer', 'ThreadSanitizer', 'LeakSanitizer'] ci-flags: [''] + if: ${{ contains(github.event_name, 'pull_request') }} runs-on: ${{ matrix.runner }} # define this for containers env: GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 - container: - # TODO(aelwazir): Adding multiple os distros - image: evuedsoacr.azurecr.io/ml-lang-comp-us/rocm-tools/rocm-rocprofiler-sdk-staging-base:latest - options: --privileged --ipc=host --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --cap-add CAP_SYS_ADMIN --security-opt seccomp=unconfined - credentials: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - steps: - uses: actions/checkout@v4 - with: - submodules: true - name: List Files shell: bash @@ -409,6 +390,7 @@ jobs: which-realpath() { echo -e "\n$1 resolves to $(realpath $(which $1))"; echo "$($(which $1) --version &> /dev/stdout | head -n 1)"; } for i in python python3 git cmake ctest; do which-realpath $i; done ls -la + cat /sys/class/kfd/kfd/topology/nodes/*/gpu_id || true; /opt/rocm/bin/rocminfo | grep gfx - name: Install requirements shell: bash @@ -432,9 +414,9 @@ jobs: shell: bash run: python3 ./source/scripts/run-ci.py -B build - --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.runner }}-${{ matrix.os }}-${{ matrix.sanitizer }} - --build-jobs 8 - --site ${{ matrix.runner }} + --name ${{ github.repository }}-${{ github.ref_name }}-${{ matrix.os }}-${{ matrix.sanitizer }} + --build-jobs 16 + --site $(echo $RUNNER_HOSTNAME)-$(/opt/rocm/bin/rocm_agent_enumerator | sed -n '2 p') --gpu-targets ${{ env.GPU_LIST }} --memcheck=${{ matrix.sanitizer }} ${{ matrix.ci-flags }} diff --git a/source/lib/common/logging.cpp b/source/lib/common/logging.cpp index 1160372888..be20fd6c06 100644 --- a/source/lib/common/logging.cpp +++ b/source/lib/common/logging.cpp @@ -116,8 +116,6 @@ update_logging(const logging_config& cfg, bool setup_env, int env_override) if(setup_env) { common::set_env("GOOGLE_LOG_DIR", get_env("PWD", ""), env_override); - common::set_env("GOOGLE_LOGTOSTDERR", cfg.loglevel, env_override); - common::set_env("GOOGLE_ALSOLOGTOSTDERR", cfg.alsologtostderr, env_override); } } } // namespace common diff --git a/source/lib/rocprofiler-sdk-tool/tool.cpp b/source/lib/rocprofiler-sdk-tool/tool.cpp index 568dc5dabe..0a6f02b5b7 100644 --- a/source/lib/rocprofiler-sdk-tool/tool.cpp +++ b/source/lib/rocprofiler-sdk-tool/tool.cpp @@ -660,10 +660,12 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/, record); auto kernel_trace_ss = std::stringstream{}; + // TODO(aelwazir): To be changed back to use node id once ROCR fixes + // the hsa_agents to use the real node id tool::csv::kernel_trace_csv_encoder::write_row( kernel_trace_ss, CHECK_NOTNULL(buffered_name_info)->kind_names.at(record->kind), - agent_info->at(record->agent_id)->node_id, + agent_info->at(record->agent_id)->logical_node_id, record->queue_id.handle, record->kernel_id, std::move(kernel_name), @@ -710,14 +712,16 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/, static_cast(header->payload); auto memory_copy_trace_ss = std::stringstream{}; + // TODO(aelwazir): To be changed back to use node id once ROCR fixes + // the hsa_agents to use the real node id tool::csv::memory_copy_csv_encoder::write_row( memory_copy_trace_ss, CHECK_NOTNULL(buffered_name_info)->kind_names.at(record->kind), CHECK_NOTNULL(buffered_name_info) ->operation_names.at(record->kind) .at(record->operation), - agent_info->at(record->src_agent_id)->node_id, - agent_info->at(record->dst_agent_id)->node_id, + agent_info->at(record->src_agent_id)->logical_node_id, + agent_info->at(record->dst_agent_id)->logical_node_id, record->correlation_id.internal, record->start_timestamp, record->end_timestamp); @@ -930,10 +934,12 @@ counter_record_callback(rocprofiler_profile_counting_dispatch_data_t dispatch_da using csv_encoder = tool::csv::counter_collection_csv_encoder; auto counter_collection_ss = std::stringstream{}; + // TODO(aelwazir): To be changed back to use node id once ROCR fixes + // the hsa_agents to use the real node id csv_encoder::write_row(counter_collection_ss, correlation_id.internal, cnt_dispatch_data_v->dispatch_index, - agent_info->at(dispatch_data.agent_id)->node_id, + agent_info->at(dispatch_data.agent_id)->logical_node_id, dispatch_data.queue_id.handle, getpid(), cnt_dispatch_data_v->thread_id, @@ -963,7 +969,9 @@ list_metrics_iterate_agents(rocprofiler_agent_version_t, { const auto* agent = static_cast(agents[idx]); auto counters_v = counter_vec_t{}; - uint32_t node_id = agent->node_id; + // TODO(aelwazir): To be changed back to use node id once ROCR fixes + // the hsa_agents to use the real node id + uint32_t node_id = agent->logical_node_id; ROCPROFILER_CALL( rocprofiler_iterate_agent_supported_counters( agent->id, diff --git a/source/lib/rocprofiler-sdk/hsa/async_copy.cpp b/source/lib/rocprofiler-sdk/hsa/async_copy.cpp index 877fc0a1c8..9882ff3c21 100644 --- a/source/lib/rocprofiler-sdk/hsa/async_copy.cpp +++ b/source/lib/rocprofiler-sdk/hsa/async_copy.cpp @@ -272,9 +272,10 @@ async_copy_handler(hsa_signal_value_t signal_value, void* arg) { ROCP_HSA_TABLE_CALL(ERROR, get_core_table()->hsa_signal_destroy_fn(_data->rocp_signal)); delete _data; - get_active_signals()->fetch_sub(1); } + get_active_signals()->fetch_sub(1); + if(_corr_id) _corr_id->sub_ref_count(); return (signal_value > 0); diff --git a/source/scripts/run-ci.py b/source/scripts/run-ci.py index fd212f582f..e92c274583 100755 --- a/source/scripts/run-ci.py +++ b/source/scripts/run-ci.py @@ -12,8 +12,8 @@ import multiprocessing # this constant is used to define CTEST_PROJECT_NAME # and default value for CTEST_SUBMIT_URL -_PROJECT_NAME = "rocprofiler-v2-internal" -_BASE_URL = "10.194.116.31/cdash" +_PROJECT_NAME = "rocprofiler-sdk-internal" +_BASE_URL = "cdash.mariadb.svc.cluster.local" _GCOVR_GENERATE_CMD = None diff --git a/tests/counter-collection/validate.py b/tests/counter-collection/validate.py index 973448ddf7..2e17b69dc0 100644 --- a/tests/counter-collection/validate.py +++ b/tests/counter-collection/validate.py @@ -42,6 +42,15 @@ def test_counter_values(input_data): f"Debug Info:\n {str(data)}" ) + for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"][ + "counter_collection" + ]: + value = itr["counter_value"] + assert int(round(value, 0)) == int(round(1 * scaling_factor, 0)), ( + f"Failure on agent " + f"{str(itr)} expected {1 * scaling_factor} but got {value} " + f"Debug Info:\n {str(data)}" + ) if __name__ == "__main__": exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) diff --git a/tests/rocprofv3/counter-collection/input1/validate.py b/tests/rocprofv3/counter-collection/input1/validate.py index 6a465a70bb..e880ea9400 100644 --- a/tests/rocprofv3/counter-collection/input1/validate.py +++ b/tests/rocprofv3/counter-collection/input1/validate.py @@ -4,6 +4,7 @@ import sys import pytest import numpy as np import pandas as pd +import re kernel_list = sorted( ["addition_kernel", "subtract_kernel", "multiply_kernel", "divide_kernel"] @@ -23,11 +24,19 @@ def test_validate_counter_collection_pmc1(input_data: pd.DataFrame): assert (df["Process_Id"].astype(int).values > 0).all() assert len(df["Kernel_Name"]) > 0 - assert kernel_list == sorted(df["Kernel_Name"].unique().tolist()) + counter_collection_pmc1_kernel_list = [ + x + for x in sorted(df["Kernel_Name"].unique().tolist()) + if not re.search(r"__amd_rocclr_.*", x) + ] + + assert kernel_list == counter_collection_pmc1_kernel_list kernel_count = dict([[itr, 0] for itr in kernel_list]) assert len(kernel_count) == len(kernel_list) for itr in df["Kernel_Name"]: + if re.search(r"__amd_rocclr_.*", itr): + continue kernel_count[itr] += 1 kn_cnt = [itr for _, itr in kernel_count.items()] assert min(kn_cnt) == max(kn_cnt) and len(unique(kn_cnt)) == 1 diff --git a/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt b/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt index 65b7f21bde..0d850dd442 100644 --- a/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt +++ b/tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt @@ -27,7 +27,7 @@ set_tests_properties( rocprofv3-test-trace-hip-in-libraries-execute PROPERTIES TIMEOUT - 45 + 100 LABELS "integration-tests" ENVIRONMENT diff --git a/tests/rocprofv3/tracing-hip-in-libraries/validate.py b/tests/rocprofv3/tracing-hip-in-libraries/validate.py index 207c599997..4a66d42d3f 100644 --- a/tests/rocprofv3/tracing-hip-in-libraries/validate.py +++ b/tests/rocprofv3/tracing-hip-in-libraries/validate.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 +import re import sys import pytest +import subprocess class dim3(object): @@ -79,7 +81,6 @@ def test_api_trace(hsa_input_data, hip_input_data): def test_kernel_trace(kernel_input_data): valid_kernel_names = sorted( [ - "__amd_rocclr_fillBufferAligned", "(anonymous namespace)::transpose(int const*, int*, int, int)", "void (anonymous namespace)::addition_kernel(float*, float const*, float const*, int, int)", "void (anonymous namespace)::divide_kernel(float*, float const*, float const*, int, int)", @@ -91,6 +92,8 @@ def test_kernel_trace(kernel_input_data): kernels = [] for row in kernel_input_data: kernel_name = row["Kernel_Name"] + if re.search(r"__amd_rocclr_.*", kernel_name): + continue assert row["Kind"] == "KERNEL_DISPATCH" assert int(row["Agent_Id"]) > 0 @@ -123,18 +126,34 @@ def test_kernel_trace(kernel_input_data): assert kernels == valid_kernel_names -def test_memory_copy_trace(memory_copy_input_data): +def test_memory_copy_trace(memory_copy_input_data, hsa_input_data): for row in memory_copy_input_data: assert row["Kind"] == "MEMORY_COPY" assert row["Direction"] in ("HOST_TO_DEVICE", "DEVICE_TO_HOST") if row["Direction"] == "HOST_TO_DEVICE": - assert int(row["Source_Agent_Id"]) == 0 + output = subprocess.check_output( + 'rocminfo | grep "Node: *' + + row["Source_Agent_Id"] + + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'', + shell=True, + ) + assert int(str(output).find("CPU")) >= 0 elif row["Direction"] == "DEVICE_TO_HOST": - assert int(row["Destination_Agent_Id"]) == 0 + output = subprocess.check_output( + 'rocminfo | grep "Node: *' + + row["Destination_Agent_Id"] + + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'', + shell=True, + ) + assert int(str(output).find("CPU")) >= 0 assert int(row["Correlation_Id"]) > 0 assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"]) - assert len(memory_copy_input_data) == 120 + valid_length = 0 + for row in hsa_input_data: + if re.search(r".*memory_async_copy.*", row["Function"]): + valid_length += 1 + assert len(memory_copy_input_data) == valid_length if __name__ == "__main__": diff --git a/tests/rocprofv3/tracing/validate.py b/tests/rocprofv3/tracing/validate.py index 2f02cd3107..cac07e4465 100644 --- a/tests/rocprofv3/tracing/validate.py +++ b/tests/rocprofv3/tracing/validate.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import sys +import subprocess import pytest @@ -67,15 +68,39 @@ def test_memory_copy_trace(memory_copy_input_data): row = memory_copy_input_data[0] assert row["Direction"] == "HOST_TO_DEVICE" - assert int(row["Source_Agent_Id"]) == 0 - assert int(row["Destination_Agent_Id"]) >= 1 + output = subprocess.check_output( + 'rocminfo | grep "Node: *' + + row["Source_Agent_Id"] + + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'', + shell=True, + ) + assert int(str(output).find("CPU")) >= 0 + output = subprocess.check_output( + 'rocminfo | grep "Node: *' + + row["Destination_Agent_Id"] + + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'', + shell=True, + ) + assert int(str(output).find("GPU")) >= 0 assert int(row["Correlation_Id"]) > 0 assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"]) row = memory_copy_input_data[1] assert row["Direction"] == "DEVICE_TO_HOST" - assert int(row["Source_Agent_Id"]) >= 1 - assert int(row["Destination_Agent_Id"]) == 0 + output = subprocess.check_output( + 'rocminfo | grep "Node: *' + + row["Source_Agent_Id"] + + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'', + shell=True, + ) + assert int(str(output).find("GPU")) >= 0 + output = subprocess.check_output( + 'rocminfo | grep "Node: *' + + row["Destination_Agent_Id"] + + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'', + shell=True, + ) + assert int(str(output).find("CPU")) >= 0 assert int(row["Correlation_Id"]) > 0 assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])