SWDEV-344154 - infrastructure to skip tests for different archs (#2850)

Change-Id: Id3bda440c4cd920ce18c14d06e2f2c6ab6346de3

tests/catch/README.md

@@ -18,7 +18,7 @@ Tests in Catch2 are declared via ```TEST_CASE```.
 Running Subtest: ctest –R “...” (Regex to match the subtest name)
 
 ## New Features
-- Skip test without recompiling tests, by addition of a json file. Default name is ```config.json``` , this can be overridden by using the variable ```HT_CONFIG_FILE=some_config.json```.
+- Skip test without recompiling tests, by addition of a json file. Default name is ```config.json``` , this can be overridden by using the variable ```HIP_CATCH_EXCLUDE_FILE=some_config.json```.
 - Json file supports regex. Ex: All tests which has the word ‘Memset’ can be skipped using ‘*Memset*’
 - Support multiple skip test list which can be set via environment variable, so you can have multiple files containing different skip test lists and can pick and choose among them depending on your platform and os.
 - Better CI integration via xunit compatible output
@@ -53,8 +53,9 @@ TEST_CASE("hipExtAPIs") {
 ## Config file schema
 Some tests can be skipped using a config file placed in hipTestMain/config folder. Multiple config files can be defined for different configurations.
 The naming convention for the file needs to be "config_platform_os_archname.json"
-Platform and os are mandatory, "all" for os if the tests needs to be skipped for all OS.
+Platform and os are mandatory.
 Arch name is optional and takes precedence while loading the json file.
+Currently the json files need to be manually chosen by the executor for the architecture of choice.
 
 example:
 config_amd_windows.json
@@ -73,7 +74,7 @@ The schema of the json file is as follows:
 ```
 
 ## Environment Variables
-- `HT_CONFIG_FILE` : This variable can be set to the config file name or full path. Disabled tests will be read from this.
+- `HIP_CATCH_EXCLUDE_FILE` : This variable can be set to the config file name or full path. Disabled tests will be read from this.
 - `HT_LOG_ENABLE` : This is for debugging the HIP Test Framework itself. Setting it to 1, all `LogPrintf` will be printed on screen
 
 ## Test Macros

tests/catch/hipTestMain/hip_test_context.cc

@@ -42,12 +42,16 @@ std::string TestContext::getMatchingConfigFile(std::string config_dir) {
     std::string arch = substringFound(amd_arch_list_, filename.filename().string());
     std::string platform = substringFound(platform_list_, filename.filename().string());
     std::string os = substringFound(os_list_, filename.filename().string());
+    std::string common_arch = "common";
+    std::vector<std::string> default_arch_vec {common_arch};
+    std::string common = substringFound(default_arch_vec, filename.filename().string());
     // if arch found then use that exit from loop
     if (arch == cur_arch) {
       configFileToUse = filename.string();
       break;
       // match the platform/os and continue to look
-    } else if ((platform == config_.platform) && (os == config_.os || os == "all")) {
+    } else if ((platform == config_.platform) && (os == config_.os || os == "all") &&
+                common == common_arch) { // ensures only common file is returned
       configFileToUse = filename.string();
     }
   }
@@ -55,7 +59,7 @@ std::string TestContext::getMatchingConfigFile(std::string config_dir) {
 }
 
 
-std::string& TestContext::getJsonFile() {
+std::string& TestContext::getCommonJsonFile() {
   fs::path config_dir = exe_path;
   config_dir = config_dir.parent_path();
   int levels = 0;
@@ -85,7 +89,7 @@ std::string& TestContext::getJsonFile() {
 }
 
 
-void TestContext::fillConfig() {
+void TestContext::getConfigFiles() {
   config_.platform = (amd ? "amd" : (nvidia ? "nvidia" : "unknown"));
   config_.os = (p_windows ? "windows" : (p_linux ? "linux" : "unknown"));
 
@@ -94,36 +98,30 @@ void TestContext::fillConfig() {
     abort();
   }
 
-  const char* env_config = std::getenv("HT_CONFIG_FILE");
+  const char* env_config = std::getenv("HIP_CATCH_EXCLUDE_FILE");
   LogPrintf("Env Config file: %s",
-            (env_config != nullptr) ? env_config : "Not found, using default config");
-
-  // Check if path has been provided
-  std::string def_config_json = "config.json";
-  std::string config_str;
+            (env_config != nullptr) ? env_config : "Not found, using common config");
+  // HIP_CATCH_EXCLUDE_FILE is set for custom file path
   if (env_config != nullptr) {
-    config_str = env_config;
+    if(fs::exists(env_config)) {
+      config_.json_files.push_back(env_config);
+    }
   } else {
-    config_str = def_config_json;
+    // get common json file
+    config_.json_files.push_back(getCommonJsonFile());
   }
 
-  fs::path config_path = config_str;
-  if (config_path.has_parent_path() && config_path.has_filename()) {
-    config_.json_file = config_str;
-  } else if (config_path.has_parent_path()) {
-    config_.json_file = config_path.string() + def_config_json;
-  } else {
-    config_.json_file = getJsonFile();
+  for (const auto& fl : config_.json_files) {
+    LogPrintf("Config file path: %s", fl.c_str());
   }
-  LogPrintf("Config file path: %s", config_.json_file.c_str());
 }
 
 TestContext::TestContext(int argc, char** argv) {
   detectOS();
   detectPlatform();
   setExePath(argc, argv);
-  fillConfig();
-  parseJsonFile();
+  getConfigFiles();
+  parseJsonFiles();
   parseOptions(argc, argv);
 }
 
@@ -161,52 +159,52 @@ bool TestContext::skipTest() const {
 
 std::string TestContext::currentPath() const { return fs::current_path().string(); }
 
-bool TestContext::parseJsonFile() {
+bool TestContext::parseJsonFiles() {
   // Check if file exists
-  if (!fs::exists(config_.json_file)) {
-    LogPrintf("Unable to find the file: %s", config_.json_file.c_str());
-    return true;
-  }
+  for (const auto& fl : config_.json_files) {
+    if (!fs::exists(fl)) {
+      LogPrintf("Unable to find the file: %s", fl.c_str());
+      return true;
+    }
+    // Open the file
+    std::ifstream js_file(fl);
+    std::string json_str((std::istreambuf_iterator<char>(js_file)), std::istreambuf_iterator<char>());
+    LogPrintf("Json contents:: %s", json_str.data());
 
-  // Open the file
-  std::ifstream js_file(config_.json_file);
-  std::string json_str((std::istreambuf_iterator<char>(js_file)), std::istreambuf_iterator<char>());
-  LogPrintf("Json contents:: %s", json_str.data());
+    picojson::value v;
+    std::string err = picojson::parse(v, json_str);
+    if (err.size() > 1) {
+      LogPrintf("Error from PicoJson: %s", err.data());
+      return false;
+    }
 
-  picojson::value v;
-  std::string err = picojson::parse(v, json_str);
-  if (err.size() > 1) {
-    LogPrintf("Error from PicoJson: %s", err.data());
-    return false;
-  }
+    if (!v.is<picojson::object>()) {
+      LogPrintf("%s", "Data in json is not in correct format, it should be an object");
+      return false;
+    }
 
-  if (!v.is<picojson::object>()) {
-    LogPrintf("%s", "Data in json is not in correct format, it should be an object");
-    return false;
-  }
+    const picojson::object& o = v.get<picojson::object>();
+    for (picojson::object::const_iterator i = o.begin(); i != o.end(); ++i) {
+      // Processing for DisabledTests
+      if (i->first == "DisabledTests") {
+        // Value should contain list of values
+        if (!i->second.is<picojson::array>()) return false;
 
-  const picojson::object& o = v.get<picojson::object>();
-  for (picojson::object::const_iterator i = o.begin(); i != o.end(); ++i) {
-    // Processing for DisabledTests
-    if (i->first == "DisabledTests") {
-      // Value should contain list of values
-      if (!i->second.is<picojson::array>()) return false;
-
-      auto& val = i->second.get<picojson::array>();
-      for (auto ai = val.begin(); ai != val.end(); ai++) {
-        std::string tmp = ai->get<std::string>();
-        std::string newRegexName;
-        for (const auto& c : tmp) {
-          if (c == '*')
-            newRegexName += ".*";
-          else
-            newRegexName += c;
+        auto& val = i->second.get<picojson::array>();
+        for (auto ai = val.begin(); ai != val.end(); ai++) {
+          std::string tmp = ai->get<std::string>();
+          std::string newRegexName;
+          for (const auto& c : tmp) {
+            if (c == '*')
+              newRegexName += ".*";
+            else
+              newRegexName += c;
+          }
+          skip_test.insert(newRegexName);
         }
-        skip_test.insert(newRegexName);
       }
     }
   }
-
   return true;
 }
 

tests/catch/include/hip_test_context.hh

@@ -63,7 +63,7 @@ static int _log_enable = (std::getenv("HT_LOG_ENABLE") ? 1 : 0);
   }
 
 typedef struct Config_ {
-  std::string json_file;  // Json file
+  std::vector<std::string> json_files;  // Json files
   std::string platform;   // amd/nvidia
   std::string os;         // windows/linux
 } Config;
@@ -99,14 +99,14 @@ class TestContext {
   std::unordered_map<std::string, rtcState> compiledKernels{};
 
   Config config_;
-  std::string& getJsonFile();
+  std::string& getCommonJsonFile();
   std::string substringFound(std::vector<std::string> list, std::string filename);
   void detectOS();
   void detectPlatform();
-  void fillConfig();
+  void getConfigFiles();
   void setExePath(int, char**);
   void parseOptions(int, char**);
-  bool parseJsonFile();
+  bool parseJsonFiles();
   std::string getMatchingConfigFile(std::string config_dir);
   const Config& getConfig() const { return config_; }