Add device mutual exclusion tests and related fixes

* Added a new test to verify mutual exclusion of access to device resources * Added some missing acquiring of mutexes to some RSMI calls, as well as try-catch blocks. Change-Id: I87aac009878a0b2d1f975e1d5b794d887bb23ff9 [ROCm/rocm_smi_lib commit: f8b57c3b16]
2020-04-07 17:13:50 -05:00
parent 49b562b209
commit 01401b0caa
10 changed files with 429 additions and 38 deletions
@@ -0,0 +1,224 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2020, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include <iostream>
+#include <string>
+#include <map>
+
+#include "gtest/gtest.h"
+#include "rocm_smi/rocm_smi.h"
+#include "rocm_smi_test/functional/mutual_exclusion.h"
+#include "rocm_smi_test/test_common.h"
+
+
+TestMutualExclusion::TestMutualExclusion() : TestBase() {
+  set_title("Mutual Exclusion Test");
+  set_description("Verify that RSMI only allows 1 process at a time"
+    " to access RSMI resources (primarily sysfs files). This test has one "
+    "process that obtains the mutex that ensures only 1 process accesses a "
+      "device's sysfs files at a time, and another process that attempts "
+      "to access the device's sysfs files. The second process should fail "
+      "in these attempts.");
+}
+
+TestMutualExclusion::~TestMutualExclusion(void) {
+}
+
+extern rsmi_status_t rsmi_test_sleep(uint32_t dv_ind, uint32_t seconds);
+
+void TestMutualExclusion::SetUp(void) {
+  std::string label;
+  rsmi_status_t ret;
+
+  //   TestBase::SetUp(RSMI_INIT_FLAG_RESRV_TEST1);
+  MakeHeaderStr(kSetupLabel, &label);
+  printf("\n\t%s\n", label.c_str());
+
+  sleeper_process_ = false;
+  child_ = 0;
+  child_ = fork();
+
+  if (child_ != 0) {
+    sleeper_process_ = true;  // sleeper_process is parent
+
+    // RSMI_INIT_FLAG_RESRV_TEST1 tells rsmi to fail immediately
+    // if it can't get the mutex instead of waiting.
+    ret = rsmi_init(RSMI_INIT_FLAG_RESRV_TEST1);
+    ASSERT_EQ(ret, RSMI_STATUS_SUCCESS);
+
+    sleep(2);  // Let both processes get through rsmi_init
+  } else {
+    sleep(1);  // Let the sleeper process get through rsmi_init() before
+              // this one goes, so it doesn't fail.
+    ret = rsmi_init(RSMI_INIT_FLAG_RESRV_TEST1);
+    ASSERT_EQ(ret, RSMI_STATUS_SUCCESS);
+
+    sleep(2);  // Let both processes get through rsmi_init;
+  }
+  return;
+}
+
+void TestMutualExclusion::DisplayTestInfo(void) {
+  TestBase::DisplayTestInfo();
+}
+
+void TestMutualExclusion::DisplayResults(void) const {
+  TestBase::DisplayResults();
+  return;
+}
+
+void TestMutualExclusion::Close() {
+  // This will close handles opened within rsmitst utility calls and call
+  // rsmi_shut_down(), so it should be done after other hsa cleanup
+  TestBase::Close();
+}
+
+extern rsmi_status_t
+rsmi_test_sleep(uint32_t dv_ind, uint32_t seconds);
+
+void TestMutualExclusion::Run(void) {
+  rsmi_status_t ret;
+
+  if (sleeper_process_) {
+    std::cout << "MUTEX_HOLDER process: started sleeping for 10 seconds..." <<
+                                                                     std::endl;
+    ret = rsmi_test_sleep(0, 10);
+    ASSERT_EQ(ret, RSMI_STATUS_SUCCESS);
+    std::cout << "MUTEX_HOLDER process: Sleep process woke up." << std::endl;
+    pid_t cpid = wait(nullptr);
+    ASSERT_EQ(cpid, child_);
+  } else {
+    // Both processes should have completed rsmi_init().
+    // let the other process get started on rsmi_test_sleep().
+    sleep(2);
+    TestBase::Run();
+    std::cout << "TESTER process: verifing that all rsmi_dev_* functions "
+                    "return RSMI_STATUS_BUSY because MUTEX_HOLDER process "
+                                               "holds the mutex" << std::endl;
+    // Try all the device related rsmi calls. They should all fail with
+    // RSMI_STATUS_BUSY
+    // Set dummy values should to working, deterministic values.
+    uint16_t dmy_ui16 = 0;
+    uint32_t dmy_ui32 = 1;
+    int32_t dmy_i32 = 0;
+    uint64_t dmy_ui64 = 0;
+    int64_t dmy_i64 = 0;
+    char dmy_str[10];
+    rsmi_dev_perf_level_t dmy_perf_lvl;
+    rsmi_frequencies_t dmy_freqs;
+    rsmi_od_volt_freq_data_t dmy_od_volt;
+    rsmi_freq_volt_region_t dmy_vlt_reg;
+    rsmi_error_count_t dmy_err_cnt;
+    rsmi_ras_err_state_t dmy_ras_err_st;
+
+    ret = rsmi_dev_id_get(0, &dmy_ui16);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_vendor_id_get(0, &dmy_ui16);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_name_get(0, dmy_str, 10);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_brand_get(0, dmy_str, 10);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_vendor_name_get(0, dmy_str, 10);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_vram_vendor_get(0, dmy_str, 10);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_serial_number_get(0, dmy_str, 10);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_subsystem_id_get(0, &dmy_ui16);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_subsystem_vendor_id_get(0, &dmy_ui16);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_unique_id_get(0, &dmy_ui64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_pci_id_get(0, &dmy_ui64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_pci_throughput_get(0, &dmy_ui64, &dmy_ui64, &dmy_ui64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_pci_replay_counter_get(0, &dmy_ui64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_pci_bandwidth_set(0, 0);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_fan_rpms_get(0, dmy_ui32, &dmy_i64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_fan_speed_get(0, 0, &dmy_i64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_fan_speed_max_get(0, 0, &dmy_ui64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_temp_metric_get(0, dmy_ui32, RSMI_TEMP_CURRENT, &dmy_i64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_fan_reset(0, 0);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_fan_speed_set(0, dmy_ui32, 0);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_perf_level_get(0, &dmy_perf_lvl);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_overdrive_level_get(0, &dmy_ui32);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_gpu_clk_freq_get(0, RSMI_CLK_TYPE_SYS, &dmy_freqs);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_od_volt_info_get(0, &dmy_od_volt);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_od_volt_curve_regions_get(0, &dmy_ui32, &dmy_vlt_reg);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_overdrive_level_set(dmy_i32, 0);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_gpu_clk_freq_set(0, RSMI_CLK_TYPE_SYS, 0);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_ecc_count_get(0, RSMI_GPU_BLOCK_UMC, &dmy_err_cnt);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_ecc_enabled_get(0, &dmy_ui64);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+    ret = rsmi_dev_ecc_status_get(0, RSMI_GPU_BLOCK_UMC, &dmy_ras_err_st);
+    ASSERT_EQ(ret, RSMI_STATUS_BUSY);
+
+    std::cout << "TESTER process: Finished verifying that all "
+                "rsmi_dev_* functions returned RSMI_STATUS_BUSY" << std::endl;
+    exit(0);
+  }
+}
@@ -0,0 +1,77 @@
+/*
+ * =============================================================================
+ *   ROC Runtime Conformance Release License
+ * =============================================================================
+ * The University of Illinois/NCSA
+ * Open Source License (NCSA)
+ *
+ * Copyright (c) 2020, Advanced Micro Devices, Inc.
+ * All rights reserved.
+ *
+ * Developed by:
+ *
+ *                 AMD Research and AMD ROC Software Development
+ *
+ *                 Advanced Micro Devices, Inc.
+ *
+ *                 www.amd.com
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal with the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ *  - Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimers.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimers in
+ *    the documentation and/or other materials provided with the distribution.
+ *  - Neither the names of <Name of Development Group, Name of Institution>,
+ *    nor the names of its contributors may be used to endorse or promote
+ *    products derived from this Software without specific prior written
+ *    permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS WITH THE SOFTWARE.
+ *
+ */
+#ifndef TESTS_ROCM_SMI_TEST_FUNCTIONAL_MUTUAL_EXCLUSION_H_
+#define TESTS_ROCM_SMI_TEST_FUNCTIONAL_MUTUAL_EXCLUSION_H_
+
+#include "rocm_smi_test/test_base.h"
+
+class TestMutualExclusion : public TestBase {
+ public:
+    TestMutualExclusion();
+
+  // @Brief: Destructor for test case of TestMutualExclusion
+  virtual ~TestMutualExclusion();
+
+  // @Brief: Setup the environment for measurement
+  virtual void SetUp();
+
+  // @Brief: Core measurement execution
+  virtual void Run();
+
+  // @Brief: Clean up and retrive the resource
+  virtual void Close();
+
+  // @Brief: Display  results
+  virtual void DisplayResults() const;
+
+  // @Brief: Display information about what this test does
+  virtual void DisplayTestInfo(void);
+
+ private:
+  bool sleeper_process_;
+  int child_;
+};
+
+#endif  // TESTS_ROCM_SMI_TEST_FUNCTIONAL_MUTUAL_EXCLUSION_H_
@@ -77,6 +77,7 @@
 #include "functional/xgmi_read_write.h"
 #include "functional/mem_page_info_read.h"
 #include "functional/api_support_read.h"
+#include "functional/mutual_exclusion.h"

 static RSMITstGlobals *sRSMIGlvalues = nullptr;

@@ -223,6 +224,14 @@ TEST(rsmitstReadOnly, TestAPISupportRead) {
  TestAPISupportRead tst;
  RunGenericTest(&tst);
 }
+TEST(rsmitstReadOnly, TestMutualExclusion) {
+  TestMutualExclusion test;
+
+  test.DisplayTestInfo();
+  test.SetUp();
+  test.Run();
+  RunCustomTestEpilog(&test);
+}

 int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
@@ -242,22 +251,5 @@ int main(int argc, char** argv) {

  int ret = 0;
  sRSMIGlvalues = &settings;
-  ret = RUN_ALL_TESTS();
-
-  if (ret) {
-    return ret;
-  }
-
-  settings.init_options = RSMI_INIT_FLAG_ALL_GPUS;
-
-  std::cout << "****************************************" << std::endl;
-  std::cout << "****************************************" << std::endl;
-  std::cout << "****************************************" << std::endl;
-  std::cout << "Re-running tests with init options: " << std::hex <<
-                                    settings.init_options << std::endl;
-  std::cout << "****************************************" << std::endl;
-  std::cout << "****************************************" << std::endl;
-  std::cout << "****************************************" << std::endl;
-  settings.init_options = RSMI_INIT_FLAG_ALL_GPUS;
  return RUN_ALL_TESTS();
 }
@@ -54,18 +54,19 @@ static const int kOutputLineLength = 80;
 static const char kLabelDelimiter[] = "####";
 static const char kDescriptionLabel[] = "TEST DESCRIPTION";
 static const char kTitleLabel[] = "TEST NAME";
-static const char kSetupLabel[] = "TEST SETUP";
 static const char kRunLabel[] = "TEST EXECUTION";
 static const char kCloseLabel[] = "TEST CLEAN UP";
 static const char kResultsLabel[] = "TEST RESULTS";

+// This one is used outside this file
+const char kSetupLabel[] = "TEST SETUP";

 TestBase::TestBase() : setup_failed_(false), description_("") {
 }
 TestBase::~TestBase() {
 }

-static void MakeHeaderStr(const char *inStr, std::string *outStr) {
+void MakeHeaderStr(const char *inStr, std::string *outStr) {
  assert(outStr != nullptr);
  assert(inStr != nullptr);

@@ -77,14 +78,19 @@ static void MakeHeaderStr(const char *inStr, std::string *outStr) {
  *outStr += kLabelDelimiter;
 }

-void TestBase::SetUp(void) {
+void TestBase::SetUp(uint64_t init_flags) {
  std::string label;
  rsmi_status_t err;

  MakeHeaderStr(kSetupLabel, &label);
  printf("\n\t%s\n", label.c_str());

-  err = rsmi_init(init_options());
+  if (init_flags) {
+    err = rsmi_init(init_flags);
+  } else {
+    err = rsmi_init(init_options());
+  }
+
  if (err != RSMI_STATUS_SUCCESS) {
    setup_failed_ = true;
  }
@@ -57,7 +57,9 @@ class TestBase {

  // @Brief: Before run the core measure codes, do something to set up
  // i.e. init runtime, prepare packet...
-  virtual void SetUp(void);
+  // The init_flags option will override any flags set for the whole test
+  // suite
+  virtual void SetUp(uint64_t init_flags = 0);

  // @Brief: Core measurement codes executing here
  virtual void Run(void);
@@ -135,4 +137,7 @@ class TestBase {
    } \
 }

+void MakeHeaderStr(const char *inStr, std::string *outStr);
+extern const char kSetupLabel[];
+
 #endif  // TESTS_ROCM_SMI_TEST_TEST_BASE_H_