AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix setup-env + hsa/rocm/ompt serialization + testing + misc (#156)

Browse Source 
- Fix setup-env.sh
  - Closes #149 
- omnitrace exe color
- test-install.sh script
- if config variable is updated in config or env, include in generated
config
- metadata for hsa, rocm, and ompt
  - Closes #153 
  - Closes #154
This commit is contained in:
Jonathan R. Madsen
2022-09-13 08:11:48 -05:00
committed by GitHub
parent 4ed8f8f762
commit 15e6e6d979
17 changed files with 499 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/bin/omnitrace-avail/avail.cpp
+5 -1
View File
@@ -802,7 +802,11 @@ write_settings_info(std::ostream& os, const array_t<bool, N>& opts,
if(category_view.count(citr) > 0) _found = true;
}
if(!print_advanced && _categories.count("settings::advanced") > 0)
_not_in_category_view.emplace(_name);
{
if(!sitr->second->get_config_updated() &&
!sitr->second->get_environ_updated())
_not_in_category_view.emplace(_name);
}
if(!_found)
{
_not_in_category_view.emplace(_name);
source/bin/omnitrace-avail/generate_config.cpp
+1
View File
@@ -62,6 +62,7 @@ ignore_setting(const Tp& _v)
{
if(_v->get_hidden()) return true;
if(exclude_setting(_v->get_env_name())) return true;
if(_v->get_config_updated() || _v->get_environ_updated()) return false;
if(!is_selected(_v->get_env_name()) && !is_selected(_v->get_name())) return true;
if(available_only && !_v->get_enabled()) return true;
if(!category_view.empty())
source/bin/omnitrace/info.hpp
+7 -3
View File
@@ -25,6 +25,7 @@
#include "fwd.hpp"
#include "module_function.hpp"
#include <timemory/log/color.hpp>
#include <timemory/mpl/policy.hpp>
#include <timemory/settings.hpp>
#include <timemory/settings/types.hpp>
@@ -81,9 +82,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
_handle_error();
else
{
verbprintf_bare(_level, "%s", ::tim::log::color::source());
verbprintf(_level, "Outputting '%s'... ", _oname.c_str());
dump_info(ofs, _data);
verbprintf_bare(_level, "Done\n");
verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end());
}
ofs.close();
}
@@ -109,9 +111,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
_handle_error();
else
{
verbprintf_bare(_level, "%s", ::tim::log::color::source());
verbprintf(_level, "Outputting '%s'... ", _oname.c_str());
ofs << oss.str() << std::endl;
verbprintf_bare(_level, "Done\n");
verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end());
}
ofs.close();
}
@@ -136,9 +139,10 @@ dump_info(const string_t& _label, string_t _oname, const string_t& _ext,
_handle_error();
else
{
verbprintf_bare(_level, "%s", ::tim::log::color::source());
verbprintf(_level, "Outputting '%s'... ", _oname.c_str());
ofs << oss.str() << std::endl;
verbprintf_bare(_level, "Done\n");
verbprintf_bare(_level, "Done\n%s", ::tim::log::color::end());
}
ofs.close();
}
source/bin/omnitrace/omnitrace.cpp
-1
View File
@@ -1790,7 +1790,6 @@ main(int argc, char** argv)
}
verbprintf(2, "Beginning instrumentation loop...\n");
verbprintf(1, "\n");
auto _report_info = [](int _lvl, const string_t& _action, const string_t& _type,
const string_t& _reason, const string_t& _name,
const std::string& _extra = {}) {
source/lib/omnitrace/library/critical_trace.cpp
+4 -2
View File
@@ -33,6 +33,7 @@
#include <timemory/backends/dmp.hpp>
#include <timemory/backends/threading.hpp>
#include <timemory/hash/types.hpp>
#include <timemory/operations/types/file_output_message.hpp>
#include <timemory/tpls/cereal/cereal/archives/json.hpp>
#include <timemory/tpls/cereal/cereal/cereal.hpp>
#include <timemory/utility/macros.hpp>
@@ -643,8 +644,9 @@ save_call_chain_json(const std::string& _fname, const std::string& _label,
if(_msg)
{
if(_func.empty()) _func = __FUNCTION__;
OMNITRACE_VERBOSE(0, "[%s] Outputting '%s'...\n", _func.c_str(),
_fname.c_str());
if(get_verbose() >= 0)
operation::file_output_message<critical_trace::call_chain>{}(
_fname, std::string{ _func });
}
std::stringstream oss{};
if(_call_chain.size() > 100000)
source/lib/omnitrace/library/defines.hpp.in
+2
View File
@@ -40,3 +40,5 @@
#define OMNITRACE_SAMPLING_GPU_TEMP OMNITRACE_SAMPLING_GPU_TEMP_idx
#define OMNITRACE_SAMPLING_GPU_BUSY OMNITRACE_SAMPLING_GPU_BUSY_idx
#define OMNITRACE_SAMPLING_GPU_MEMORY_USAGE OMNITRACE_SAMPLING_GPU_MEMORY_USAGE_idx
#define OMNITRACE_METADATA(...) ::tim::manager::add_metadata(__VA_ARGS__)
source/lib/omnitrace/library/gpu.cpp
+160
View File
@@ -21,6 +21,10 @@
// SOFTWARE.
#include "library/gpu.hpp"
#include "library/debug.hpp"
#include "library/defines.hpp"
#include <timemory/manager.hpp>
#if defined(OMNITRACE_USE_ROCM_SMI) && OMNITRACE_USE_ROCM_SMI > 0
# include "library/rocm_smi.hpp"
@@ -37,6 +41,23 @@
# define OMNITRACE_USE_HIP 0
#endif
#if OMNITRACE_USE_HIP > 0
# include <hip/hip_runtime.h>
# include <hip/hip_runtime_api.h>
# if !defined(OMNITRACE_HIP_RUNTIME_CALL)
# define OMNITRACE_HIP_RUNTIME_CALL(err) \
{ \
if(err != ::tim::hip::success_v && (int) err != 0) \
{ \
OMNITRACE_THROW( \
"[%s:%d] Warning! HIP API call failed with code %i :: %s\n", \
__FILE__, __LINE__, (int) err, hipGetErrorString(err)); \
} \
}
# endif
#endif
namespace omnitrace
{
namespace gpu
@@ -64,5 +85,144 @@ device_count()
return 0;
#endif
}
template <typename ArchiveT>
void
add_hip_device_metadata(ArchiveT& ar)
{
#if OMNITRACE_USE_HIP > 0
namespace cereal = tim::cereal;
using cereal::make_nvp;
using intvec_t = std::vector<int>;
int _device_count = 0;
int _current_device = 0;
hipError_t _device_count_err = hipGetDeviceCount(&_device_count);
if(_device_count_err != hipSuccess) return;
hipError_t _current_device_err = hipGetDevice(&_current_device);
scope::destructor _dtor{ [_current_device, _current_device_err]() {
if(_current_device_err == hipSuccess)
{
OMNITRACE_HIP_RUNTIME_CALL(hipSetDevice(_current_device));
}
} };
if(_current_device_err != hipSuccess || _device_count == 0) return;
# define OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(NAME) \
ar(make_nvp(#NAME, _device_prop.NAME));
# define OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY(NAME, ...) \
ar(make_nvp(NAME, __VA_ARGS__));
ar.setNextName("hip_device_properties");
ar.startNode();
ar.makeArray();
scope::destructor _prop_dtor{ [&ar]() { ar.finishNode(); } };
for(int dev = 0; dev < _device_count; ++dev)
{
auto _device_prop = hipDeviceProp_t{};
int _driver_version = 0;
int _runtime_version = 0;
OMNITRACE_HIP_RUNTIME_CALL(hipSetDevice(dev));
OMNITRACE_HIP_RUNTIME_CALL(hipGetDeviceProperties(&_device_prop, dev));
OMNITRACE_HIP_RUNTIME_CALL(hipDriverGetVersion(&_driver_version));
OMNITRACE_HIP_RUNTIME_CALL(hipRuntimeGetVersion(&_runtime_version));
ar.startNode();
ar(make_nvp("name", std::string{ _device_prop.name }));
ar(make_nvp("driver_version", _driver_version));
ar(make_nvp("runtime_version", _runtime_version));
ar(make_nvp("capability.major_version", _device_prop.major));
ar(make_nvp("capability.minor_version", _device_prop.minor));
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(totalGlobalMem)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(totalConstMem)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(clockRate)
# if OMNITRACE_HIP_VERSION >= 5000
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memoryClockRate)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memoryBusWidth)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(l2CacheSize)
# endif
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(sharedMemPerBlock)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(regsPerBlock)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(warpSize)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(multiProcessorCount)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxThreadsPerMultiProcessor)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxThreadsPerBlock)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY(
"maxThreadsDim",
intvec_t{ _device_prop.maxThreadsDim[0], _device_prop.maxThreadsDim[1],
_device_prop.maxThreadsDim[2] })
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP_ARRAY("maxGridSize",
intvec_t{ _device_prop.maxGridSize[0],
_device_prop.maxGridSize[1],
_device_prop.maxGridSize[2] })
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(memPitch)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(textureAlignment)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(kernelExecTimeoutEnabled)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(integrated)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(canMapHostMemory)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(ECCEnabled)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(cooperativeLaunch)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(cooperativeMultiDeviceLaunch)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciDomainID)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciBusID)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pciDeviceID)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(computeMode)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(computeMode)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(gcnArch)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(gcnArchName)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(isMultiGpuBoard)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(clockInstructionRate)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pageableMemoryAccess)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(pageableMemoryAccessUsesHostPageTables)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(directManagedMemAccessFromHost)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(concurrentManagedAccess)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(concurrentKernels)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(maxSharedMemoryPerMultiProcessor)
OMNITRACE_SERIALIZE_HIP_DEVICE_PROP(asicRevision)
const char* _compute_mode_descr[] = {
"Default (multiple host threads can use ::hipSetDevice() with device "
"simultaneously)",
"Exclusive (only one host thread in one process is able to use "
"::hipSetDevice() with this device)",
"Prohibited (no host thread can use ::hipSetDevice() with this device)",
"Exclusive Process (many threads in one process is able to use "
"::hipSetDevice() with this device)",
"Unknown",
nullptr
};
ar(make_nvp("computeModeDescription",
std::string{ _compute_mode_descr[_device_prop.computeMode] }));
ar.finishNode();
}
#else
(void) ar;
#endif
}
void
add_hip_device_metadata()
{
if(device_count() == 0) return;
OMNITRACE_METADATA([](auto& ar) {
try
{
add_hip_device_metadata(ar);
} catch(std::runtime_error& _e)
{
OMNITRACE_VERBOSE(2, "%s\n", _e.what());
}
});
}
} // namespace gpu
} // namespace omnitrace
source/lib/omnitrace/library/gpu.hpp
+3
View File
@@ -31,5 +31,8 @@ device_count();
int
hip_device_count();
void
add_hip_device_metadata();
} // namespace gpu
} // namespace omnitrace
source/lib/omnitrace/library/ompt.cpp
+27 -2
View File
@@ -88,8 +88,33 @@ shutdown()
} // namespace ompt
} // namespace omnitrace
// include the ompt_start_tool definition
# include <timemory/components/ompt/start_tool.hpp>
extern "C" ompt_start_tool_result_t*
ompt_start_tool(unsigned int omp_version, const char* runtime_version)
{
TIMEMORY_PRINTF(stderr, "OpenMP version: %u, runtime version: %s\n", omp_version,
runtime_version);
OMNITRACE_METADATA("OMP_VERSION", omp_version);
OMNITRACE_METADATA("OMP_RUNTIME_VERSION", runtime_version);
static auto ompt_initialize = [](ompt_function_lookup_t lookup,
int initial_device_num,
ompt_data_t* tool_data) -> int {
TIMEMORY_PRINTF(stderr, "OpenMP-tools configuring for initial device %i\n\n",
initial_device_num);
tim::ompt::configure<TIMEMORY_OMPT_API_TAG>(lookup, initial_device_num,
tool_data);
return 1; // success
};
static auto ompt_finalize = [](ompt_data_t* tool_data) {
TIMEMORY_PRINTF(stderr, "OpenMP-tools finalized\n\n");
tim::consume_parameters(tool_data);
};
static auto data = ompt_start_tool_result_t{ ompt_initialize, ompt_finalize, { 0 } };
return (ompt_start_tool_result_t*) &data;
}
#else
namespace omnitrace
source/lib/omnitrace/library/rocm.cpp
+6
View File
@@ -308,6 +308,12 @@ extern "C"
"failed! OMNITRACE_ROCPROFILER_LIBRARY=%s\n",
_rocprof.filename.c_str());
}
else
{
HsaRsrcFactory::Instance().PrintGpuAgents("ROCm");
}
gpu::add_hip_device_metadata();
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Loading... %s\n",
(_success) ? "Done" : "Failed");
source/lib/omnitrace/library/rocprofiler/hsa_rsrc_factory.cpp
+29 -65
View File
@@ -22,6 +22,7 @@ THE SOFTWARE.
#include "library/rocprofiler/hsa_rsrc_factory.hpp"
#include "library/debug.hpp"
#include "library/defines.hpp"
#include <timemory/manager.hpp>
@@ -778,73 +779,36 @@ HsaRsrcFactory::LoadAndFinalize(const AgentInfo* agent_info, const char* brig_pa
bool
HsaRsrcFactory::PrintGpuAgents(const std::string&)
{
std::cout << std::flush;
// std::clog << header << " :" << std::endl;
char key[1024], value[1024];
const AgentInfo* agent_info;
int size = uint32_t(gpu_list_.size());
for(int idx = 0; idx < size; idx++)
std::vector<AgentInfo> _agents = {};
for(const auto* itr : gpu_list_)
{
agent_info = gpu_list_[idx];
/* std::clog << "> agent[" << idx << "] :" << std::endl;
std::clog << ">> Name : " << agent_info->name << std::endl;
std::clog << ">> APU : " << agent_info->is_apu << std::endl;
std::clog << ">> HSAIL profile : " << agent_info->profile << std::endl;
std::clog << ">> Max Wave Size : " << agent_info->max_wave_size << std::endl;
std::clog << ">> Max Queue Size : " << agent_info->max_queue_size <<
std::endl; std::clog << ">> CU number : " << agent_info->cu_num << std::endl;
std::clog << ">> Waves per CU : " << agent_info->waves_per_cu << std::endl;
std::clog << ">> SIMDs per CU : " << agent_info->simds_per_cu << std::endl;
std::clog << ">> SE number : " << agent_info->se_num << std::endl;
std::clog << ">> Shader Arrays per SE : " << agent_info->shader_arrays_per_se
<< std::endl;
*/
#define OMNITRACE_METADATA(...) tim::manager::add_metadata(__VA_ARGS__)
sprintf(key, "ROCM_AGENT_%d_NAME", idx);
sprintf(value, "%s", agent_info->name);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_IS_APU", idx);
sprintf(value, "%d", static_cast<int>(agent_info->is_apu));
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_HSA_PROFILE", idx);
sprintf(value, "%d", agent_info->profile);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_MAX_WAVE_SIZE", idx);
sprintf(value, "%d", agent_info->max_wave_size);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_MAX_QUEUE_SIZE", idx);
sprintf(value, "%d", agent_info->max_queue_size);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_CU_NUMBER", idx);
sprintf(value, "%d", agent_info->cu_num);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_WAVES_PER_CU", idx);
sprintf(value, "%d", agent_info->waves_per_cu);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_SIMDs_PER_CU", idx);
sprintf(value, "%d", agent_info->simds_per_cu);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_SE_NUMBER", idx);
sprintf(value, "%d", agent_info->se_num);
OMNITRACE_METADATA(key, value);
sprintf(key, "ROCM_AGENT_%d_SHADER_ARRAYS_PER_SE", idx);
sprintf(value, "%d", agent_info->shader_arrays_per_se);
OMNITRACE_METADATA(key, value);
if(itr) _agents.emplace_back(*itr);
}
OMNITRACE_METADATA([_agents](auto& ar) {
namespace cereal = ::tim::cereal;
ar.setNextName("rocm_agents");
ar.startNode();
ar.makeArray();
for(auto itr : _agents)
{
ar.startNode();
ar(cereal::make_nvp("name", std::string{ itr.name }),
cereal::make_nvp("is_apu", itr.is_apu),
cereal::make_nvp("hsa_profile", itr.profile),
cereal::make_nvp("max_wave_size", itr.max_wave_size),
cereal::make_nvp("max_queue_size", itr.max_queue_size),
cereal::make_nvp("cu_number", itr.cu_num),
cereal::make_nvp("waves_per_cu", itr.waves_per_cu),
cereal::make_nvp("simds_per_cu", itr.simds_per_cu),
cereal::make_nvp("se_num", itr.se_num),
cereal::make_nvp("shader_arrays_per_se", itr.shader_arrays_per_se));
ar.finishNode();
}
ar.finishNode();
});
return true;
}