AI/rocm-systems
2
0
Forkuj 0
Kod Zgłoszenia Oczekujące zmiany Actions Packages Projekty Wydania Wiki Aktywność

wsl/hsakmt: initial commit

Przeglądaj źródła 
Signed-off-by: lyndonli <Lyndon.Li@amd.com>
Signed-off-by: Horatio Zhang <Hongkun.Zhang@amd.com>
Signed-off-by: Shi.Leslie <Yuliang.Shi@amd.com>
Signed-off-by: LonglongYao <Longlong.Yao@amd.com>
Signed-off-by: tiancyin <tianci.yin@amd.com>
Signed-off-by: Frank Min <Frank.Min@amd.com>
Signed-off-by: Aaron Liu <aaron.liu@amd.com>
Signed-off-by: Shane Xiao <shane.xiao@amd.com>
Signed-off-by: Lang Yu <lang.yu@amd.com>
Signed-off-by: Feifei Xu <Feifei.Xu@amd.com>
Signed-off-by: Ruili Ji <ruiliji2@amd.com>
Signed-off-by: Qiang Yu <qiang.yu@amd.com>
Signed-off-by: Flora Cui <flora.cui@amd.com>
This commit is contained in:
Flora Cui
2024-04-30 09:01:09 -05:00
zatwierdzone przez Frank Min
commit 79a63cf292
48 zmienionych plików z 13833 dodań i 0 usunięć
Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files
util/atomic_helpers.h
+519
Wyświetl plik
@@ -0,0 +1,519 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
/*
Helpers to use native types with C++11 atomic operations.
Fixes GCC builtin functionality for x86 with respect to WC and non-temporal
stores.
*/
#ifndef HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_
#define HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_
#include <atomic>
#include "utils.h"
//ALWAYS_CONSERVATIVE will very likely overfence your code.
//For use as a debugging aid only.
#define ALWAYS_CONSERVATIVE 0
#if !ALWAYS_CONSERVATIVE
#if defined(__x86_64__) || defined(_M_X64)
#define X64_ORDER_WC 1
#endif
#if X64_ORDER_WC
#include <xmmintrin.h>
#endif
#endif
namespace rocr {
namespace atomic {
static constexpr int c11ToBuiltInFlags(std::memory_order order)
{
#if ALWAYS_CONSERVATIVE
return __ATOMIC_RELAXED;
#elif X64_ORDER_WC
return __ATOMIC_RELAXED;
#else
return (order == std::memory_order_relaxed) ? __ATOMIC_RELAXED :
(order == std::memory_order_acquire) ? __ATOMIC_ACQUIRE :
(order == std::memory_order_release) ? __ATOMIC_RELEASE :
(order == std::memory_order_seq_cst) ? __ATOMIC_SEQ_CST :
(order == std::memory_order_consume) ? __ATOMIC_CONSUME :
(order == std::memory_order_acq_rel) ? __ATOMIC_ACQ_REL :
__ATOMIC_SEQ_CST;
#endif
}
static __forceinline void PreFence(std::memory_order order) {
#if ALWAYS_CONSERVATIVE
switch (order) {
case std::memory_order_release:
case std::memory_order_seq_cst:
case std::memory_order_acq_rel:
__atomic_thread_fence(__ATOMIC_SEQ_CST);
default:;
}
#elif X64_ORDER_WC
switch (order) {
case std::memory_order_release:
case std::memory_order_seq_cst:
case std::memory_order_acq_rel:
_mm_sfence();
default:;
}
#endif
}
static __forceinline void PostFence(std::memory_order order) {
#if ALWAYS_CONSERVATIVE
switch (order) {
case std::memory_order_seq_cst:
case std::memory_order_acq_rel:
case std::memory_order_acquire:
__atomic_thread_fence(__ATOMIC_SEQ_CST);
default:;
}
#elif X64_ORDER_WC
switch (order) {
case std::memory_order_seq_cst:
return _mm_mfence();
case std::memory_order_acq_rel:
case std::memory_order_acquire:
return _mm_lfence();
default:;
}
#endif
}
static __forceinline void Fence(std::memory_order order=std::memory_order_seq_cst) {
#if ALWAYS_CONSERVATIVE
__atomic_thread_fence(__ATOMIC_SEQ_CST);
#elif X64_ORDER_WC
switch (order) {
case std::memory_order_seq_cst:
case std::memory_order_acq_rel:
return _mm_mfence();
case std::memory_order_acquire:
return _mm_lfence();
case std::memory_order_release:
return _mm_sfence();
default:;
}
#else
std::atomic_thread_fence(order);
#endif
}
template <class T>
static __forceinline void BasicCheck(const T* ptr) {
constexpr bool value = __atomic_always_lock_free(sizeof(T), 0);
static_assert(value, "Atomic type may not be compatible with peripheral atomics.");
};
template <class T>
static __forceinline void BasicCheck(const volatile T* ptr) {
constexpr bool value = __atomic_always_lock_free(sizeof(T), 0);
static_assert(value, "Atomic type may not be compatible with peripheral atomics.");
};
/// @brief: Load value of type T atomically with specified memory order.
/// @param: ptr(Input), a pointer to type T.
/// @param: order(Input), memory order with atomic load, relaxed by default.
/// @return: T, loaded value.
template <class T>
static __forceinline T
Load(const T* ptr, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
T ret;
PreFence(order);
__atomic_load(ptr, &ret, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: function overloading, for more info, see previous one.
/// @param: ptr(Input), a pointer to volatile type T.
/// @param: order(Input), memory order with atomic load, relaxed by default.
/// @return: T, loaded value.
template <class T>
static __forceinline T
Load(const volatile T* ptr,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
T ret;
PreFence(order);
__atomic_load(ptr, &ret, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Store value of type T with specified memory order.
/// @param: ptr(Input), a pointer to instance which will be stored.
/// @param: val(Input), value to be stored.
/// @param: order(Input), memory order with atomic store, relaxed by default.
/// @return: void.
template <class T>
static __forceinline void Store(
T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
__atomic_store(ptr, &val, c11ToBuiltInFlags(order));
PostFence(order);
}
/// @brief: Function overloading, for more info, see previous one.
/// @param: ptr(Input), a pointer to volatile instance which will be stored.
/// @param: val(Input), value to be stored.
/// @param: order(Input), memory order with atomic store, relaxed by default.
/// @return: void.
template <class T>
static __forceinline void Store(
volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
__atomic_store(ptr, &val, c11ToBuiltInFlags(order));
PostFence(order);
}
/// @brief: Compare and swap value atomically with specified memory order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value to be stored if condition is satisfied.
/// @param: expected(Input), value which is expected.
/// @param: order(Input), memory order with atomic operation.
/// @return: T, observed value of type T.
template <class T>
static __forceinline T
Cas(T* ptr, T val, T expected,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
__atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED);
PostFence(order);
return expected;
}
/// @brief: Function overloading, for more info, see previous one.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: val(Input), value to be stored if condition is satisfied.
/// @param: expected(Input), value which is expected.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, observed value of type T.
template <class T>
static __forceinline T
Cas(volatile T* ptr, T val, T expected,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
__atomic_compare_exchange(ptr, &expected, &val, false, c11ToBuiltInFlags(order), __ATOMIC_RELAXED);
PostFence(order);
return expected;
}
/// @brief: Exchange the value atomically with specified memory order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value to be stored.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, the value prior to the exchange.
template <class T>
static __forceinline T
Exchange(T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
T ret;
PreFence(order);
__atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Function overloading, for more info, see previous one.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value to be stored.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, the value prior to the exchange.
template <class T>
static __forceinline T
Exchange(volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
T ret;
PreFence(order);
__atomic_exchange(ptr, &val, &ret, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Add value to variable atomically with specified memory order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value to be added.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, the value of the variable prior to the addition.
template <class T>
static __forceinline T
Add(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Subtract value from the variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value to be subtraced.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of the variable prior to the subtraction.
template <class T>
static __forceinline T
Sub(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Bit And operation on variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value which is ANDed with variable.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
And(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Bit Or operation on variable atomically with specified memory order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value which is ORed with variable.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
Or(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Bit Xor operation on variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: val(Input), value which is XORed with variable.
/// @order: order(Input), memory order which is relaxed by default.
/// @return: T, valud of variable prior to the opertaion.
template <class T>
static __forceinline T
Xor(T* ptr, T val, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Increase the value of variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
Increment(T* ptr, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Decrease the value of the variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to variable which is operated on.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
Decrement(T* ptr, std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Add value to variable atomically with specified memory order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: val(Input), value to be added.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, the value of the variable prior to the addition.
template <class T>
static __forceinline T
Add(volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_add(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Subtract value from the variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: val(Input), value to be subtraced.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of the variable prior to the subtraction.
template <class T>
static __forceinline T
Sub(volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_sub(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Bit And operation on variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: val(Input), value which is ANDed with variable.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
And(volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_and(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Bit Or operation on variable atomically with specified memory order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: val(Input), value which is ORed with variable.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T Or(volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_or(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Bit Xor operation on variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: val(Input), value which is XORed with variable.
/// @order: order(Input), memory order which is relaxed by default.
/// @return: T, valud of variable prior to the opertaion.
template <class T>
static __forceinline T
Xor(volatile T* ptr, T val,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_xor(ptr, val, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Increase the value of variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
Increment(volatile T* ptr,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_add(ptr, 1, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
/// @brief: Decrease the value of the variable atomically with specified memory
/// order.
/// @param: ptr(Input), a pointer to volatile variable which is operated on.
/// @param: order(Input), memory order which is relaxed by default.
/// @return: T, value of variable prior to the operation.
template <class T>
static __forceinline T
Decrement(volatile T* ptr,
std::memory_order order = std::memory_order_relaxed) {
BasicCheck<T>(ptr);
PreFence(order);
T ret = __atomic_fetch_sub(ptr, 1, c11ToBuiltInFlags(order));
PostFence(order);
return ret;
}
} // namespace atomic
} // namespace rocr
#ifdef X64_ORDER_WC
#undef X64_ORDER_WC
#endif
#ifdef ALWAYS_CONSERVATIVE
#undef ALWAYS_CONSERVATIVE
#endif
#endif // HSA_RUNTIME_CORE_UTIL_ATOMIC_HELPERS_H_
util/flag.cpp
+226
Wyświetl plik
@@ -0,0 +1,226 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#include "core/util/flag.h"
#include "core/util/utils.h"
#include "core/util/os.h"
#include <vector>
#include <map>
#include <string>
#include <algorithm>
#include <locale>
namespace rocr {
FILE* log_file = stderr;
uint8_t log_flags[8];
void log_printf(const char* file, int line, const char* format, ...) {
va_list ap;
std::stringstream str_thrd_id;
str_thrd_id << std::hex << std::this_thread::get_id();
va_start(ap, format);
char message[4096];
vsnprintf(message, sizeof(message), format, ap);
va_end(ap);
fprintf(log_file, ":%-25s:%-4d: %010lld us: [pid:%-5d tid:0x%s] [***rocr***] %s\n",
file, line, os::ReadAccurateClock()/1000ULL, os::GetProcessId(),
str_thrd_id.str().c_str(), message);
fflush(log_file);
}
// split at separators
static std::vector<std::string> split(std::string& str, char sep) {
std::vector<std::string> ret;
while (!str.empty()) {
size_t pos = str.find(sep);
if (pos == std::string::npos) {
ret.push_back(str);
return ret;
}
ret.push_back(str.substr(0, pos));
str.erase(0, pos + 1);
}
return ret;
};
// Parse id,id-id,... strings into id lists
static std::vector<uint32_t> get_elements(std::string& str, uint32_t maxElement) {
std::vector<uint32_t> ret;
MAKE_NAMED_SCOPE_GUARD(error, [&]() { ret.clear(); });
std::vector<std::string> ranges = split(str, ',');
for (auto& str : ranges) {
auto range = split(str, '-');
// failure, too many -'s.
if (range.size() > 2) return ret;
char* end;
uint32_t index = strtoul(range[0].c_str(), &end, 10);
// Invalid syntax - id's must be base 10 digits only.
if (*end != '\0') return ret;
if (index <= maxElement) ret.push_back(index);
if (range.size() == 2) {
uint32_t secondindex = strtoul(range[1].c_str(), &end, 10);
if (*end != '\0') return ret; // bad syntax
if (secondindex < index) return ret; // inverted range
secondindex = Min(secondindex, maxElement);
for (uint32_t i = index + 1; i < secondindex + 1; i++) ret.push_back(i);
}
}
// Confirm no duplicate ids.
std::sort(ret.begin(), ret.end());
if (std::adjacent_find(ret.begin(), ret.end()) != ret.end()) return ret;
// Good parse, keep result.
error.Dismiss();
return ret;
};
/*
Parse env var per the following syntax, all whitespace is ignored:
ID = [0-9][0-9]* ex. base 10 numbers
ID_list = (ID | ID-ID)[, (ID | ID-ID)]* ex. 0,2-4,7
GPU_list = ID_list ex. 0,2-4,7
CU_list = 0x[0-F]* | ID_list ex. 0x337F OR 0,2-4,7
CU_Set = GPU_list : CU_list ex. 0,2-4,7:0-15,32-47 OR 0,2-4,7:0x337F
HSA_CU_MASK = CU_Set [; CU_Set]* ex. 0,2-4,7:0-15,32-47; 3-9:0x337F
GPU indexes are taken post ROCR_VISIBLE_DEVICES reordering.
Listed or bit set CUs will be enabled at queue creation on the associated GPU.
All other CUs on the associated GPUs will be disabled.
CU masks of unlisted GPUs are not restricted.
Repeating a GPU or CU ID is a syntax error.
Parsing stops at the first CU_Set that has a syntax error, that set and all
following sets are ignored.
Specifying a mask with no usable CUs (CU_list is 0x0) is a syntax error.
Users should use ROCR_VISIBLE_DEVICES if they want to exclude use of a
particular GPU.
*/
void Flag::parse_masks(std::string& var, uint32_t maxGpu, uint32_t maxCU) {
if (var.empty()) return;
// Remove whitespace
auto end = std::remove_if(var.begin(), var.end(),
[](char c) { return std::isspace<char>(c, std::locale::classic()); });
var.erase(end, var.end());
// Switch to uppercase
for (auto& c : var) c = toupper(c);
// Iterate over cu sets
auto sets = split(var, ';');
for (auto& set : sets) {
auto parts = split(set, ':');
if (parts.size() != 2) return;
// temp storage for cu_set parsing.
std::vector<uint32_t> gpu_index;
std::vector<uint32_t> mask;
// parse cu list first, check for bitmask format
if (parts[1][1] == 'X') {
// Confirm hex format and strip prefix
auto& cu = parts[1];
if (cu[0] != '0') return;
cu.erase(0, 2);
// Ensure all valid hex characters
for (auto& c : cu) {
if (!isxdigit(c)) return;
}
// Convert to uint32_t, lsb first.
size_t len = cu.length();
while (len != 0) {
size_t trim = Min(len, size_t(8));
len -= trim;
auto tmp = cu.substr(len, trim);
auto chunk = stoul(tmp, nullptr, 16);
mask.push_back(chunk);
}
// Trim dwords beyond maxCUs
uint32_t maxDwords = maxCU / 32 + 1;
if (maxDwords < mask.size()) mask.resize(maxDwords);
// Trim leading zeros
while (!mask.empty() && mask.back() == 0) mask.pop_back();
// Mask 0x0 is an error.
if (mask.empty()) return;
} else {
// parse cu lists
auto cu_indices = get_elements(parts[1], maxCU);
if (cu_indices.empty()) return;
uint32_t maxdword = cu_indices.back() / 32 + 1;
mask.resize(maxdword, 0);
for (auto id : cu_indices) {
uint32_t index, offset;
index = id / 32;
offset = id % 32;
mask[index] |= 1ul << offset;
}
}
// parse device list
gpu_index = get_elements(parts[0], maxGpu);
if (gpu_index.empty()) return;
// Ensure that no GPU was repeated across cu_sets
for (auto id : gpu_index) {
if (cu_mask_.find(id) != cu_mask_.end()) return;
}
// Insert into map
for (auto id : gpu_index) {
cu_mask_[id] = mask;
}
}
}
} // namespace rocr
util/flag.h
+360
Wyświetl plik
@@ -0,0 +1,360 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2021, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef HSA_RUNTIME_CORE_INC_FLAG_H_
#define HSA_RUNTIME_CORE_INC_FLAG_H_
#include <stdint.h>
#include <vector>
#include <map>
#include <string>
#include "core/util/os.h"
#include "core/util/utils.h"
namespace rocr {
class Flag {
public:
enum SDMA_OVERRIDE { SDMA_DISABLE, SDMA_ENABLE, SDMA_DEFAULT };
enum SRAMECC_ENABLE { SRAMECC_DISABLED, SRAMECC_ENABLED, SRAMECC_DEFAULT };
// The values are meaningful and chosen to satisfy the thunk API.
enum XNACK_REQUEST { XNACK_DISABLE = 0, XNACK_ENABLE = 1, XNACK_UNCHANGED = 2 };
static_assert(XNACK_DISABLE == 0, "XNACK_REQUEST enum values improperly changed.");
static_assert(XNACK_ENABLE == 1, "XNACK_REQUEST enum values improperly changed.");
// Lift limit for 2.10 release RCCL workaround.
const size_t DEFAULT_SCRATCH_SINGLE_LIMIT = 146800640; // small_limit >> 2;
explicit Flag() { Refresh(); }
virtual ~Flag() {}
void Refresh() {
std::string var = os::GetEnvVar("HSA_CHECK_FLAT_SCRATCH");
check_flat_scratch_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_ENABLE_VM_FAULT_MESSAGE");
enable_vm_fault_message_ = (var == "0") ? false : true;
var = os::GetEnvVar("HSA_ENABLE_QUEUE_FAULT_MESSAGE");
enable_queue_fault_message_ = (var == "0") ? false : true;
var = os::GetEnvVar("HSA_ENABLE_INTERRUPT");
enable_interrupt_ = (var == "0") ? false : true;
var = os::GetEnvVar("HSA_ENABLE_SDMA");
enable_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
var = os::GetEnvVar("HSA_ENABLE_PEER_SDMA");
enable_peer_sdma_ = (var == "0") ? SDMA_DISABLE : ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
var = os::GetEnvVar("HSA_ENABLE_SDMA_GANG");
enable_sdma_gang_ = (var == "0") ? SDMA_DISABLE :
((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
var = os::GetEnvVar("HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE");
enable_sdma_copy_size_override_ = (var == "0") ? SDMA_DISABLE :
((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
visible_gpus_ = os::GetEnvVar("ROCR_VISIBLE_DEVICES");
filter_visible_gpus_ = os::IsEnvVarSet("ROCR_VISIBLE_DEVICES");
var = os::GetEnvVar("HSA_RUNNING_UNDER_VALGRIND");
running_valgrind_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_SDMA_WAIT_IDLE");
sdma_wait_idle_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_MAX_QUEUES");
max_queues_ = static_cast<uint32_t>(atoi(var.c_str()));
// Maximum amount of scratch mem that can be used per process per gpu
var = os::GetEnvVar("HSA_SCRATCH_MEM");
scratch_mem_size_ = atoi(var.c_str());
// Scratch memory sizes > HSA_SCRATCH_SINGLE_LIMIT will trigger a use-once scheme
// We also reserve HSA_SCRATCH_SINGLE_LIMIT per process per gpu to guarrantee we
// have sufficient memory to for scratch in case user tried to allocate all device
// memory
if (os::IsEnvVarSet("HSA_SCRATCH_SINGLE_LIMIT")) {
var = os::GetEnvVar("HSA_SCRATCH_SINGLE_LIMIT");
scratch_single_limit_ = atoi(var.c_str());
} else {
scratch_single_limit_ = DEFAULT_SCRATCH_SINGLE_LIMIT;
}
tools_lib_names_ = os::GetEnvVar("HSA_TOOLS_LIB");
var = os::GetEnvVar("HSA_TOOLS_REPORT_LOAD_FAILURE");
ifdebug {
report_tool_load_failures_ = (var == "1") ? true : false;
} else {
report_tool_load_failures_ = (var == "0") ? false : true;
}
var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR");
disable_fragment_alloc_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH");
enable_sdma_hdp_flush_ = (var == "0") ? false : true;
var = os::GetEnvVar("HSA_REV_COPY_DIR");
rev_copy_dir_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_FORCE_FINE_GRAIN_PCIE");
fine_grain_pcie_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_NO_SCRATCH_RECLAIM");
no_scratch_reclaim_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_NO_SCRATCH_THREAD_LIMITER");
no_scratch_thread_limit_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_DISABLE_IMAGE");
disable_image_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_DISABLE_PC_SAMPLING");
disable_pc_sampling_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_LOADER_ENABLE_MMAP_URI");
loader_enable_mmap_uri_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_FORCE_SDMA_SIZE");
force_sdma_size_ = var.empty() ? 1024 * 1024 : atoi(var.c_str());
var = os::GetEnvVar("HSA_IGNORE_SRAMECC_MISREPORT");
check_sramecc_validity_ = (var == "1") ? false : true;
// Legal values are zero "0" or one "1". Any other value will
// be interpreted as not defining the env variable.
var = os::GetEnvVar("HSA_XNACK");
xnack_ = (var == "0") ? XNACK_DISABLE : ((var == "1") ? XNACK_ENABLE : XNACK_UNCHANGED);
var = os::GetEnvVar("HSA_ENABLE_DEBUG");
debug_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_CU_MASK_SKIP_INIT");
cu_mask_skip_init_ = (var == "1") ? true : false;
// Temporary opt-in for corrected HSA_AMD_AGENT_INFO_COOPERATIVE_COMPUTE_UNIT_COUNT behavior.
// Will become opt-out and possibly removed in future releases.
var = os::GetEnvVar("HSA_COOP_CU_COUNT");
coop_cu_count_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_DISCOVER_COPY_AGENTS");
discover_copy_agents_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_SVM_PROFILE");
svm_profile_ = var;
var = os::GetEnvVar("HSA_ENABLE_SRAMECC");
sramecc_enable_ =
(var == "0") ? SRAMECC_DISABLED : ((var == "1") ? SRAMECC_ENABLED : SRAMECC_DEFAULT);
var = os::GetEnvVar("HSA_IMAGE_PRINT_SRD");
image_print_srd_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_ENABLE_MWAITX");
enable_mwaitx_ = (var == "1") ? true : false;
// Temporary environment variable to disable CPU affinity override
// Will either rename to HSA_OVERRIDE_CPU_AFFINITY later or remove completely.
var = os::GetEnvVar("HSA_OVERRIDE_CPU_AFFINITY_DEBUG");
override_cpu_affinity_ = (var == "0") ? false : true;
}
void parse_masks(uint32_t maxGpu, uint32_t maxCU) {
std::string var = os::GetEnvVar("HSA_CU_MASK");
parse_masks(var, maxGpu, maxCU);
}
bool check_flat_scratch() const { return check_flat_scratch_; }
bool enable_vm_fault_message() const { return enable_vm_fault_message_; }
bool enable_queue_fault_message() const { return enable_queue_fault_message_; }
bool enable_interrupt() const { return enable_interrupt_; }
bool enable_sdma_hdp_flush() const { return enable_sdma_hdp_flush_; }
bool running_valgrind() const { return running_valgrind_; }
bool sdma_wait_idle() const { return sdma_wait_idle_; }
bool report_tool_load_failures() const { return report_tool_load_failures_; }
bool disable_fragment_alloc() const { return disable_fragment_alloc_; }
bool rev_copy_dir() const { return rev_copy_dir_; }
bool fine_grain_pcie() const { return fine_grain_pcie_; }
bool no_scratch_reclaim() const { return no_scratch_reclaim_; }
bool no_scratch_thread_limiter() const { return no_scratch_thread_limit_; }
SDMA_OVERRIDE enable_sdma() const { return enable_sdma_; }
SDMA_OVERRIDE enable_peer_sdma() const { return enable_peer_sdma_; }
SDMA_OVERRIDE enable_sdma_gang() const { return enable_sdma_gang_; }
SDMA_OVERRIDE enable_sdma_copy_size_override() const { return enable_sdma_copy_size_override_; }
std::string visible_gpus() const { return visible_gpus_; }
bool filter_visible_gpus() const { return filter_visible_gpus_; }
uint32_t max_queues() const { return max_queues_; }
size_t scratch_mem_size() const { return scratch_mem_size_; }
size_t scratch_single_limit() const { return scratch_single_limit_; }
std::string tools_lib_names() const { return tools_lib_names_; }
bool disable_image() const { return disable_image_; }
bool disable_pc_sampling() const { return disable_pc_sampling_; }
bool loader_enable_mmap_uri() const { return loader_enable_mmap_uri_; }
size_t force_sdma_size() const { return force_sdma_size_; }
bool check_sramecc_validity() const { return check_sramecc_validity_; }
bool override_cpu_affinity() const { return override_cpu_affinity_; }
bool image_print_srd() const { return image_print_srd_; }
bool check_mwaitx(bool mwaitx_supported) {
if (enable_mwaitx_ && !mwaitx_supported) enable_mwaitx_ = false;
return enable_mwaitx_;
}
XNACK_REQUEST xnack() const { return xnack_; }
bool debug() const { return debug_; }
const std::vector<uint32_t>& cu_mask(uint32_t gpu_index) const {
static const std::vector<uint32_t> empty;
auto it = cu_mask_.find(gpu_index);
if (it == cu_mask_.end()) return empty;
return it->second;
}
bool cu_mask_skip_init() const { return cu_mask_skip_init_; }
bool coop_cu_count() const { return coop_cu_count_; }
bool discover_copy_agents() const { return discover_copy_agents_; }
const std::string& svm_profile() const { return svm_profile_; }
SRAMECC_ENABLE sramecc_enable() const { return sramecc_enable_; }
private:
bool check_flat_scratch_;
bool enable_vm_fault_message_;
bool enable_interrupt_;
bool enable_sdma_hdp_flush_;
bool running_valgrind_;
bool sdma_wait_idle_;
bool enable_queue_fault_message_;
bool report_tool_load_failures_;
bool disable_fragment_alloc_;
bool rev_copy_dir_;
bool fine_grain_pcie_;
bool no_scratch_reclaim_;
bool no_scratch_thread_limit_;
bool disable_image_;
bool disable_pc_sampling_;
bool loader_enable_mmap_uri_;
bool check_sramecc_validity_;
bool debug_;
bool cu_mask_skip_init_;
bool coop_cu_count_;
bool discover_copy_agents_;
bool override_cpu_affinity_;
bool image_print_srd_;
bool enable_mwaitx_;
SDMA_OVERRIDE enable_sdma_;
SDMA_OVERRIDE enable_peer_sdma_;
SDMA_OVERRIDE enable_sdma_gang_;
SDMA_OVERRIDE enable_sdma_copy_size_override_;
bool filter_visible_gpus_;
std::string visible_gpus_;
uint32_t max_queues_;
size_t scratch_mem_size_;
size_t scratch_single_limit_;
std::string tools_lib_names_;
std::string svm_profile_;
size_t force_sdma_size_;
// Indicates user preference for Xnack state.
XNACK_REQUEST xnack_;
SRAMECC_ENABLE sramecc_enable_;
// Map GPU index post RVD to its default cu mask.
std::map<uint32_t, std::vector<uint32_t>> cu_mask_;
void parse_masks(std::string& args, uint32_t maxGpu, uint32_t maxCU);
DISALLOW_COPY_AND_ASSIGN(Flag);
};
} // namespace rocr
#endif // header guard
util/lazy_ptr.h
+155
Wyświetl plik
@@ -0,0 +1,155 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIESd OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_
#define HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_
#include <memory>
#include <utility>
#include <functional>
#include "core/util/locks.h"
#include "core/util/utils.h"
namespace rocr {
/*
* Wrapper for a std::unique_ptr that initializes its object at first use.
*/
template <typename T> class lazy_ptr {
public:
lazy_ptr() {}
explicit lazy_ptr(std::function<T*()> Constructor) { reset(Constructor); }
lazy_ptr(lazy_ptr&& rhs) {
obj = std::move(rhs.obj);
func = std::move(rhs.func);
}
lazy_ptr& operator=(lazy_ptr&& rhs) {
obj = std::move(rhs.obj);
func = std::move(rhs.func);
}
lazy_ptr(lazy_ptr&) = delete;
lazy_ptr& operator=(lazy_ptr&) = delete;
void reset(std::function<T*()> Constructor = nullptr) {
obj.reset();
func = Constructor;
}
void reset(T* ptr) {
obj.reset(ptr);
func = nullptr;
}
bool operator==(T* rhs) const { return obj.get() == rhs; }
bool operator!=(T* rhs) const { return obj.get() != rhs; }
const std::unique_ptr<T>& operator->() const {
make(true);
assert(obj != nullptr && "Null dereference through lazy_ptr.");
return obj;
}
std::unique_ptr<T>& operator*() {
make(true);
return obj;
}
const std::unique_ptr<T>& operator*() const {
make(true);
return obj;
}
/*
* Ensures that the object is created or is being created.
* This is useful when early construction of the object is required.
*/
void touch() const { make(false); }
// Tells if the lazy object has been constructed or not.
// Construction may fail silently (return nullptr).
bool created() const {
std::atomic_thread_fence(std::memory_order_acquire);
return func == nullptr;
}
// Tells if the lazy object exists or not.
bool empty() const {
std::atomic_thread_fence(std::memory_order_acquire);
return obj == nullptr;
}
private:
mutable std::unique_ptr<T> obj;
mutable std::function<T*(void)> func;
mutable KernelMutex lock;
// Separated from make to improve inlining.
void make_body(bool block) const {
if (block) {
lock.Acquire();
} else if (!lock.Try()) {
return;
}
MAKE_SCOPE_GUARD([&]() { lock.Release(); });
if (func == nullptr) return;
T* ptr = func();
obj.reset(ptr);
std::atomic_thread_fence(std::memory_order_release);
func = nullptr;
}
__forceinline void make(bool block) const {
if (!created()) {
make_body(block);
}
}
};
} // namespace rocr
#endif // HSA_RUNTIME_CORE_UTIL_LAZY_PTR_H_
util/lnx/os_linux.cpp
+771
Wyświetl plik
@@ -0,0 +1,771 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#ifdef __linux__
#include "core/util/os.h"
#include "core/util/utils.h"
#include <link.h>
#include <dlfcn.h>
#include <pthread.h>
#include <limits.h>
#include <sched.h>
#include <sys/sysinfo.h>
#include <sys/time.h>
#include <sys/utsname.h>
#include <unistd.h>
#include <errno.h>
#include <cstring>
#include <atomic>
#include <memory>
#include <string>
#include <utility>
#include <semaphore.h>
#include "core/inc/runtime.h"
#if defined(__i386__) || defined(__x86_64__)
#include <cpuid.h>
#endif
namespace rocr {
namespace os {
struct ThreadArgs {
void* entry_args;
ThreadEntry entry_function;
};
void* __stdcall ThreadTrampoline(void* arg) {
ThreadArgs* ar = (ThreadArgs*)arg;
ThreadEntry CallMe = ar->entry_function;
void* Data = ar->entry_args;
delete ar;
CallMe(Data);
return nullptr;
}
// Thread container allows multiple waits and separate close (destroy).
class os_thread {
public:
explicit os_thread(ThreadEntry function, void* threadArgument, uint stackSize)
: thread(0), lock(nullptr), state(RUNNING) {
int err;
std::unique_ptr<ThreadArgs> args(new ThreadArgs);
lock = CreateMutex();
if (lock == nullptr) return;
args->entry_args = threadArgument;
args->entry_function = function;
pthread_attr_t attrib;
err = pthread_attr_init(&attrib);
if (err != 0) {
fprintf(stderr, "pthread_attr_init failed: %s\n", strerror(err));
return;
}
if (stackSize != 0) {
stackSize = Max(uint(PTHREAD_STACK_MIN), stackSize);
stackSize = AlignUp(stackSize, 4096);
err = pthread_attr_setstacksize(&attrib, stackSize);
if (err != 0) {
fprintf(stderr, "pthread_attr_setstacksize failed: %s\n", strerror(err));
err = pthread_attr_destroy(&attrib);
if (err != 0) {
fprintf(stderr, "pthread_attr_destroy failed: %s\n", strerror(err));
return;
}
}
}
int cores = 0;
cpu_set_t* cpuset = nullptr;
if (core::Runtime::runtime_singleton_->flag().override_cpu_affinity()) {
cores = get_nprocs_conf();
cpuset = CPU_ALLOC(cores);
if (cpuset == nullptr) {
fprintf(stderr, "CPU_ALLOC failed: %s\n", strerror(errno));
return;
}
CPU_ZERO_S(CPU_ALLOC_SIZE(cores), cpuset);
for (int i = 0; i < cores; i++) {
CPU_SET_S(i, CPU_ALLOC_SIZE(cores), cpuset);
}
err = pthread_attr_setaffinity_np(&attrib, CPU_ALLOC_SIZE(cores), cpuset);
CPU_FREE(cpuset);
if (err != 0) {
fprintf(stderr, "pthread_setaffinity_np failed: %s\n", strerror(err));
return;
}
}
err = pthread_create(&thread, &attrib, ThreadTrampoline, args.get());
// Probably a stack size error since system limits can be different from PTHREAD_STACK_MIN
// Attempt to grow the stack within reason.
if ((err == EINVAL) && stackSize != 0) {
while (stackSize < 20 * 1024 * 1024) {
stackSize *= 2;
err = pthread_attr_setstacksize(&attrib, stackSize);
if (err != 0) {
fprintf(stderr, "pthread_attr_setstacksize failed: %s\n", strerror(err));
return;
}
err = pthread_create(&thread, &attrib, ThreadTrampoline, args.get());
if (err != EINVAL) break;
debug_print("pthread_create returned EINVAL, doubling stack size\n");
}
}
if (err == 0)
args.release();
else
thread = 0;
err = pthread_attr_destroy(&attrib);
if (err != 0) {
fprintf(stderr, "pthread_attr_destroy failed: %s\n", strerror(err));
}
}
os_thread(os_thread&& rhs) {
thread = rhs.thread;
lock = rhs.lock;
state = int(rhs.state);
rhs.thread = 0;
rhs.lock = nullptr;
}
os_thread(os_thread&) = delete;
~os_thread() {
if (lock != nullptr) DestroyMutex(lock);
if ((state == RUNNING) && (thread != 0)) {
int err = pthread_detach(thread);
if (err != 0) fprintf(stderr, "pthread_detach failed: %s\n", strerror(err));
}
}
bool Valid() { return (lock != nullptr) && (thread != 0); }
bool Wait() {
if (state == FINISHED) return true;
AcquireMutex(lock);
if (state == FINISHED) {
ReleaseMutex(lock);
return true;
}
int err = pthread_join(thread, NULL);
bool success = (err == 0);
if (success) state = FINISHED;
ReleaseMutex(lock);
return success;
}
private:
pthread_t thread;
Mutex lock;
std::atomic<int> state;
enum { FINISHED = 0, RUNNING = 1 };
};
static_assert(sizeof(LibHandle) == sizeof(void*), "OS abstraction size mismatch");
static_assert(sizeof(Semaphore) == sizeof(sem_t*), "OS abstraction size mismatch");
static_assert(sizeof(Mutex) == sizeof(pthread_mutex_t*), "OS abstraction size mismatch");
static_assert(sizeof(SharedMutex) == sizeof(pthread_rwlock_t*), "OS abstraction size mismatch");
static_assert(sizeof(Thread) == sizeof(os_thread*), "OS abstraction size mismatch");
LibHandle LoadLib(std::string filename) {
void* ret = dlopen(filename.c_str(), RTLD_LAZY);
if (ret == nullptr) debug_print("LoadLib(%s) failed: %s\n", filename.c_str(), dlerror());
return *(LibHandle*)&ret;
}
void* GetExportAddress(LibHandle lib, std::string export_name) {
void* ret = dlsym(*(void**)&lib, export_name.c_str());
// dlsym searches the given library and all the library's load dependencies.
// Remaining code limits symbol lookup to only the library handle given.
// This lookup pattern matches Windows.
if (ret == NULL) return ret;
link_map* map;
int err = dlinfo(*(void**)&lib, RTLD_DI_LINKMAP, &map);
if (err == -1) {
fprintf(stderr, "dlinfo failed: %s\n", dlerror());
return nullptr;
}
Dl_info info;
err = dladdr(ret, &info);
if (err == 0) {
fprintf(stderr, "dladdr failed.\n");
return nullptr;
}
if (strcmp(info.dli_fname, map->l_name) == 0) return ret;
return NULL;
}
void CloseLib(LibHandle lib) { dlclose(*(void**)&lib); }
/*
* @brief Look for a symbol called "HSA_AMD_TOOL_PRIORITY" across all loaded
* shared libraries, and if found, store the name of the library
*
* @param[in]: info A dl_phdr_info struct pointer, which contains information
* about library's load address, header, and name.
*
* @param[in]: size integer size of dl_phdr_info struct
*
* @param[out]: data copy of the data argument to dl_phdr_iterate call
*
* @retval:: Return 0 on Success. If callback returns a non-zero value,
* dl_iterate_phdr() will stop processing, even if there are unprocessed
* shared objects.
*/
static int callback(struct dl_phdr_info* info, size_t size, void* data) {
std::vector<std::string>* loadedToolsLib = (std::vector<std::string>*)data;
assert(loadedToolsLib != nullptr);
/*
* Check if lib name is not empty and its not a "vdso.so" lib,
* The vDSO is a special shared object file that is built into the Linux kernel.
* It is not a regular shared library and thus does not have all the properties
* of regular shared libraries. The way the vDSO is loaded and organized in memory
* is different from regular shared libraries and it's not guaranteed that it
* will have a specific segment or section. Hence its skipped.
*/
if ((info) && (info->dlpi_name[0] != '\0')) {
if (std::string(info->dlpi_name).find("vdso.so") != std::string::npos) return 0;
/*
* Iterate through the program headers of the loaded lib and check for PT_DYNAMIC program
* header. If the PT_DYNAMIC program header is found, use dlpi_addr and dlpi_phdr members
* of dl_phdr_info struct to get the address of the dynamic section of the loaded
* library in memory
*/
for (int i = 0; i < info->dlpi_phnum; i++) {
if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {
Elf64_Dyn* dyn_section = (Elf64_Dyn*)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
char* strings = nullptr;
Elf64_Xword limit = 0;
/*
* The dynamic section is searched for DT_STRTAB (address of string table),
* and DT_STRSZ (size of string table)
* DT_NULL - Marks the end of the _DYNAMIC array
*/
for (int j = 0;; j++) {
if (dyn_section[j].d_tag == DT_NULL) break;
if (dyn_section[j].d_tag == DT_STRTAB) strings = (char*)(dyn_section[j].d_un.d_ptr);
if (dyn_section[j].d_tag == DT_STRSZ) limit = dyn_section[j].d_un.d_val;
}
if (strings == nullptr) debug_print("String table not found");
/*
* Hacky lookup, if string and symbol tables are found,
* iterate through the strings in string table and check if
* any string matches "HSA_AMD_TOOL_PRIORITY".
* If yes, then add the name of the library to the vector of
* lib names
*/
if (strings != nullptr) {
char* end = strings + limit;
while (strings < end) {
if (strcmp(strings, "HSA_AMD_TOOL_PRIORITY") == 0) {
loadedToolsLib->push_back(info->dlpi_name);
return 0;
}
strings += (strlen(strings) + 1);
}
}
}
}
}
return 0;
}
std::vector<LibHandle> GetLoadedToolsLib() {
std::vector<LibHandle> ret;
std::vector<std::string> names;
/* Iterate through all of the loaded shared libraries in the process */
dl_iterate_phdr(callback, &names);
if (!names.empty()) {
for (auto& name : names) ret.push_back(LoadLib(name));
}
return ret;
}
std::string GetLibraryName(LibHandle lib) {
link_map *map;
if(dlinfo(lib, RTLD_DI_LINKMAP, &map)!=0)
return "";
return map->l_name;
}
Semaphore CreateSemaphore() {
sem_t *sem = new sem_t;
sem_init(sem, 0, 0);
return *(Semaphore*)&sem;
}
bool WaitSemaphore(Semaphore sem) {
while(sem_wait(*(sem_t**)&sem))
if (errno != EINTR) return false;
return true;
}
void PostSemaphore(Semaphore sem) {
if (sem_post(*(sem_t**)&sem))
assert(false && "Failed to post semaphore");
}
void DestroySemaphore(Semaphore sem) {
sem_destroy(*(sem_t**)&sem);
delete *(sem_t**)&sem;
}
Mutex CreateMutex() {
pthread_mutex_t* mutex = new pthread_mutex_t;
pthread_mutex_init(mutex, NULL);
return *(Mutex*)&mutex;
}
bool TryAcquireMutex(Mutex lock) {
return pthread_mutex_trylock(*(pthread_mutex_t**)&lock) == 0;
}
bool AcquireMutex(Mutex lock) {
return pthread_mutex_lock(*(pthread_mutex_t**)&lock) == 0;
}
void ReleaseMutex(Mutex lock) {
pthread_mutex_unlock(*(pthread_mutex_t**)&lock);
}
void DestroyMutex(Mutex lock) {
pthread_mutex_destroy(*(pthread_mutex_t**)&lock);
delete *(pthread_mutex_t**)&lock;
}
void Sleep(int delay_in_millisec) { usleep(delay_in_millisec * 1000); }
void uSleep(int delayInUs) { usleep(delayInUs); }
void YieldThread() { sched_yield(); }
Thread CreateThread(ThreadEntry function, void* threadArgument, uint stackSize) {
os_thread* result = new os_thread(function, threadArgument, stackSize);
if (!result->Valid()) {
delete result;
return nullptr;
}
return reinterpret_cast<Thread>(result);
}
void CloseThread(Thread thread) { delete reinterpret_cast<os_thread*>(thread); }
bool WaitForThread(Thread thread) { return reinterpret_cast<os_thread*>(thread)->Wait(); }
bool WaitForAllThreads(Thread* threads, uint threadCount) {
for (uint i = 0; i < threadCount; i++) WaitForThread(threads[i]);
return true;
}
bool IsEnvVarSet(std::string env_var_name) {
char* buff = NULL;
buff = getenv(env_var_name.c_str());
return (buff != NULL);
}
void SetEnvVar(std::string env_var_name, std::string env_var_value) {
setenv(env_var_name.c_str(), env_var_value.c_str(), 1);
}
int GetProcessId() {
return ::getpid();
}
std::string GetEnvVar(std::string env_var_name) {
char* buff;
buff = getenv(env_var_name.c_str());
std::string ret;
if (buff) {
ret = buff;
}
return ret;
}
size_t GetUserModeVirtualMemorySize() {
#ifdef _LP64
// https://www.kernel.org/doc/Documentation/x86/x86_64/mm.txt :
// user space is 0000000000000000 - 00007fffffffffff (=47 bits)
return (size_t)(0x800000000000);
#else
return (size_t)(0xffffffff); // ~4GB
#endif
}
size_t GetUsablePhysicalHostMemorySize() {
struct sysinfo info = {0};
if (sysinfo(&info) != 0) {
return 0;
}
const size_t physical_size =
static_cast<size_t>(info.totalram * info.mem_unit);
return std::min(GetUserModeVirtualMemorySize(), physical_size);
}
uintptr_t GetUserModeVirtualMemoryBase() { return (uintptr_t)0; }
// Os event implementation
typedef struct EventDescriptor_ {
pthread_cond_t event;
pthread_mutex_t mutex;
bool state;
bool auto_reset;
} EventDescriptor;
EventHandle CreateOsEvent(bool auto_reset, bool init_state) {
EventDescriptor* eventDescrp;
eventDescrp = (EventDescriptor*)malloc(sizeof(EventDescriptor));
pthread_mutex_init(&eventDescrp->mutex, NULL);
pthread_cond_init(&eventDescrp->event, NULL);
eventDescrp->auto_reset = auto_reset;
eventDescrp->state = init_state;
EventHandle handle = reinterpret_cast<EventHandle>(eventDescrp);
return handle;
}
int DestroyOsEvent(EventHandle event) {
if (event == NULL) {
return -1;
}
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
int ret_code = pthread_cond_destroy(&eventDescrp->event);
ret_code |= pthread_mutex_destroy(&eventDescrp->mutex);
free(eventDescrp);
return ret_code;
}
int WaitForOsEvent(EventHandle event, unsigned int milli_seconds) {
if (event == NULL) {
return -1;
}
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
// Event wait time is 0 and state is non-signaled, return directly
if (milli_seconds == 0) {
int tmp_ret = pthread_mutex_trylock(&eventDescrp->mutex);
if (tmp_ret == EBUSY) {
// Timeout
return 1;
}
}
int ret_code = 0;
pthread_mutex_lock(&eventDescrp->mutex);
if (!eventDescrp->state) {
if (milli_seconds == 0) {
ret_code = 1;
} else {
struct timespec ts;
struct timeval tp;
ret_code = gettimeofday(&tp, NULL);
ts.tv_sec = tp.tv_sec;
ts.tv_nsec = tp.tv_usec * 1000;
unsigned int sec = milli_seconds / 1000;
unsigned int mSec = milli_seconds % 1000;
ts.tv_sec += sec;
ts.tv_nsec += mSec * 1000000;
// More then one second, add 1 sec to the tv_sec elem
if (ts.tv_nsec > 1000000000) {
ts.tv_sec += 1;
ts.tv_nsec = ts.tv_nsec - 1000000000;
}
ret_code =
pthread_cond_timedwait(&eventDescrp->event, &eventDescrp->mutex, &ts);
// Time out
if (ret_code == 110) {
ret_code = 0x14003; // 1 means time out in HSA
}
if (ret_code == 0 && eventDescrp->auto_reset) {
eventDescrp->state = false;
}
}
} else if (eventDescrp->auto_reset) {
eventDescrp->state = false;
}
pthread_mutex_unlock(&eventDescrp->mutex);
return ret_code;
}
int SetOsEvent(EventHandle event) {
if (event == NULL) {
return -1;
}
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
int ret_code = 0;
ret_code = pthread_mutex_lock(&eventDescrp->mutex);
eventDescrp->state = true;
ret_code = pthread_mutex_unlock(&eventDescrp->mutex);
ret_code |= pthread_cond_signal(&eventDescrp->event);
return ret_code;
}
int ResetOsEvent(EventHandle event) {
if (event == NULL) {
return -1;
}
EventDescriptor* eventDescrp = reinterpret_cast<EventDescriptor*>(event);
int ret_code = 0;
ret_code = pthread_mutex_lock(&eventDescrp->mutex);
eventDescrp->state = false;
ret_code = pthread_mutex_unlock(&eventDescrp->mutex);
return ret_code;
}
static double invPeriod = 0.0;
uint64_t ReadAccurateClock() {
if (invPeriod == 0.0) AccurateClockFrequency();
timespec time;
int err = clock_gettime(CLOCK_MONOTONIC_RAW, &time);
if (err != 0) {
perror("clock_gettime(CLOCK_MONOTONIC_RAW,...) failed");
abort();
}
return (uint64_t(time.tv_sec) * 1000000000ull + uint64_t(time.tv_nsec)) * invPeriod;
}
uint64_t AccurateClockFrequency() {
static clockid_t clock = CLOCK_MONOTONIC;
static std::atomic<bool> first(true);
// Check kernel version - not a concurrency concern.
// use non-RAW for getres due to bug in older 2.6.x kernels
if (first.load(std::memory_order_acquire)) {
utsname kernelInfo;
if (uname(&kernelInfo) == 0) {
try {
std::string ver = kernelInfo.release;
size_t idx;
int major = std::stoi(ver, &idx);
int minor = std::stoi(ver.substr(idx + 1));
if ((major >= 4) && (minor >= 4)) {
clock = CLOCK_MONOTONIC_RAW;
}
} catch (...) {
// Kernel version string doesn't conform to the standard pattern.
// Keep using the "safe" (non-RAW) clock.
}
}
first.store(false, std::memory_order_release);
}
timespec time;
int err = clock_getres(clock, &time);
if (err != 0) {
perror("clock_getres failed");
abort();
}
if (time.tv_sec != 0 || time.tv_nsec >= 0xFFFFFFFF) {
fprintf(stderr,
"clock_getres(CLOCK_MONOTONIC(_RAW),...) returned very low "
"frequency (<1Hz).\n");
abort();
}
if (invPeriod == 0.0) invPeriod = 1.0 / double(time.tv_nsec);
return 1000000000ull / uint64_t(time.tv_nsec);
}
SharedMutex CreateSharedMutex() {
pthread_rwlockattr_t attrib;
int err = pthread_rwlockattr_init(&attrib);
if (err != 0) {
fprintf(stderr, "rw lock attribute init failed: %s\n", strerror(err));
return nullptr;
}
#ifdef __GLIBC__
err = pthread_rwlockattr_setkind_np(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
if (err != 0) {
fprintf(stderr, "Set rw lock attribute failure: %s\n", strerror(err));
return nullptr;
}
#else
err = pthread_rwlockattr_setkind(&attrib, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
if (err != 0) {
fprintf(stderr, "Set rw lock attribute failure: %s\n", strerror(err));
return nullptr;
}
#endif
pthread_rwlock_t* lock = new pthread_rwlock_t;
err = pthread_rwlock_init(lock, &attrib);
if (err != 0) {
fprintf(stderr, "rw lock init failed: %s\n", strerror(err));
return nullptr;
}
pthread_rwlockattr_destroy(&attrib);
return lock;
}
bool TryAcquireSharedMutex(SharedMutex lock) {
int err = pthread_rwlock_trywrlock(*(pthread_rwlock_t**)&lock);
return err == 0;
}
bool AcquireSharedMutex(SharedMutex lock) {
int err = pthread_rwlock_wrlock(*(pthread_rwlock_t**)&lock);
return err == 0;
}
void ReleaseSharedMutex(SharedMutex lock) {
int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock);
if (err != 0) {
fprintf(stderr, "SharedMutex unlock failed: %s\n", strerror(err));
abort();
}
}
bool TrySharedAcquireSharedMutex(SharedMutex lock) {
int err = pthread_rwlock_tryrdlock(*(pthread_rwlock_t**)&lock);
return err == 0;
}
bool SharedAcquireSharedMutex(SharedMutex lock) {
int err = pthread_rwlock_rdlock(*(pthread_rwlock_t**)&lock);
return err == 0;
}
void SharedReleaseSharedMutex(SharedMutex lock) {
int err = pthread_rwlock_unlock(*(pthread_rwlock_t**)&lock);
if (err != 0) {
fprintf(stderr, "SharedMutex unlock failed: %s\n", strerror(err));
abort();
}
}
void DestroySharedMutex(SharedMutex lock) {
pthread_rwlock_destroy(*(pthread_rwlock_t**)&lock);
delete *(pthread_rwlock_t**)&lock;
}
static uint64_t sys_clock_period_ = 0;
uint64_t ReadSystemClock() {
struct timespec ts;
clock_gettime(CLOCK_BOOTTIME, &ts);
uint64_t time = (uint64_t(ts.tv_sec) * 1000000000 + uint64_t(ts.tv_nsec));
if (sys_clock_period_ != 1)
return time / sys_clock_period_;
else
return time;
}
uint64_t SystemClockFrequency() {
struct timespec ts;
clock_getres(CLOCK_BOOTTIME, &ts);
sys_clock_period_ = (uint64_t(ts.tv_sec) * 1000000000 + uint64_t(ts.tv_nsec));
return 1000000000 / sys_clock_period_;
}
bool ParseCpuID(cpuid_t* cpuinfo) {
#if defined(__i386__) || defined(__x86_64__)
uint32_t eax, ebx, ecx, edx, max_eax = 0;
memset(cpuinfo, 0, sizeof(*cpuinfo));
/* Make sure current CPU supports at least EAX 4 */
if (!__get_cpuid_max(0x80000004, NULL)) return false;
// Manufacturer ID is a twelve-character ASCII string stored in order EBX, EDX, ECX.
if (!__get_cpuid(0, &max_eax, (uint32_t*)&cpuinfo->ManufacturerID[0],
(uint32_t*)&cpuinfo->ManufacturerID[8],
(uint32_t*)&cpuinfo->ManufacturerID[4])) {
return false;
}
if (!strcmp(cpuinfo->ManufacturerID, "AuthenticAMD")) {
if (__get_cpuid(0x80000001, &eax, &ebx, &ecx, &edx)) {
cpuinfo->mwaitx = !!((ecx >> 29) & 0x1);
}
}
return true;
#else
return false;
#endif
}
} // namespace os
} // namespace rocr
#endif
util/locks.h
+290
Wyświetl plik
@@ -0,0 +1,290 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
// Library of syncronization primitives - to be added to as needed.
#ifndef HSA_RUNTIME_CORE_UTIL_LOCKS_H_
#define HSA_RUNTIME_CORE_UTIL_LOCKS_H_
#include "utils.h"
#include "os.h"
namespace rocr {
class HybridMutex {
public:
HybridMutex():lock_(0) {
sem_ = os::CreateSemaphore();
}
~HybridMutex() {
os::DestroySemaphore(sem_);
}
bool Try() {
int old = 0;
return lock_.compare_exchange_strong(old, 1);
}
bool Acquire() {
int cnt = maxSpinIterPause + maxSpinIterYield;
int old = 0;
while (!lock_.compare_exchange_strong(old, 1)) {
cnt--;
if (cnt > maxSpinIterPause) {
_mm_pause();
} else if (cnt-- > maxSpinIterYield) {
os::YieldThread();
} else {
os::WaitSemaphore(sem_);
cnt = maxSpinIterPause + maxSpinIterYield;
}
old = 0;
}
return true;
}
void Release() {
int old = 1;
if (lock_.compare_exchange_strong(old, 0))
os::PostSemaphore(sem_);
}
private:
std::atomic<int> lock_;
os::Semaphore sem_;
const uint32_t maxSpinIterPause = 55;
const uint32_t maxSpinIterYield = 55;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(HybridMutex);
};
/// @brief: a class represents a kernel mutex.
/// Uses the kernel's scheduler to keep the waiting thread from being scheduled
/// until the lock is released (Best for long waits, though anything using
/// a kernel object is a long wait).
class KernelMutex {
public:
KernelMutex() { lock_ = os::CreateMutex(); }
~KernelMutex() { os::DestroyMutex(lock_); }
bool Try() { return os::TryAcquireMutex(lock_); }
bool Acquire() { return os::AcquireMutex(lock_); }
void Release() { os::ReleaseMutex(lock_); }
private:
os::Mutex lock_;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(KernelMutex);
};
/// @brief: represents a spin lock.
/// For very short hold durations on the order of the thread scheduling
/// quanta or less.
class SpinMutex {
public:
SpinMutex() { lock_ = 0; }
bool Try() {
int old = 0;
return lock_.compare_exchange_strong(old, 1);
}
bool Acquire() {
int old = 0;
while (!lock_.compare_exchange_strong(old, 1))
{
old=0;
os::YieldThread();
}
return true;
}
void Release() { lock_ = 0; }
private:
std::atomic<int> lock_;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(SpinMutex);
};
class KernelEvent {
public:
KernelEvent() { evt_ = os::CreateOsEvent(true, true); }
~KernelEvent() { os::DestroyOsEvent(evt_); }
bool IsSet() { return os::WaitForOsEvent(evt_, 0)==0; }
bool WaitForSet() { return os::WaitForOsEvent(evt_, 0xFFFFFFFF)==0; }
void Set() { os::SetOsEvent(evt_); }
void Reset() { os::ResetOsEvent(evt_); }
private:
os::EventHandle evt_;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(KernelEvent);
};
/// @brief: represents a yielding shared mutex.
/// aka read/write mutex
class KernelSharedMutex {
public:
/// @brief: Interfaces ScopedAcquire to shared operations.
class Shared {
public:
explicit Shared(KernelSharedMutex* lock) : lock_(lock) {}
bool Try() { return lock_->TryShared(); }
bool Acquire() { return lock_->AcquireShared(); }
void Release() { lock_->ReleaseShared(); }
private:
KernelSharedMutex* lock_;
};
KernelSharedMutex() { lock_ = os::CreateSharedMutex(); }
~KernelSharedMutex() { os::DestroySharedMutex(lock_); }
// Exclusive mode operations
bool Try() { return os::TryAcquireSharedMutex(lock_); }
bool Acquire() { return os::AcquireSharedMutex(lock_); }
void Release() { os::ReleaseSharedMutex(lock_); }
// Shared mode operations
bool TryShared() { return os::TrySharedAcquireSharedMutex(lock_); }
bool AcquireShared() { return os::SharedAcquireSharedMutex(lock_); }
void ReleaseShared() { os::SharedReleaseSharedMutex(lock_); }
// Return shared operations interface
Shared shared() { return Shared(this); }
private:
os::SharedMutex lock_;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(KernelSharedMutex);
};
/// @brief: Type trait to identify mutex types
template <class T> class isMutex {
public:
enum { value = false };
};
template <> class isMutex<HybridMutex> {
public:
enum { value = true };
};
template <> class isMutex<KernelMutex> {
public:
enum { value = true };
};
template <> class isMutex<SpinMutex> {
public:
enum { value = true };
};
template <> class isMutex<KernelSharedMutex> {
public:
enum { value = true };
};
/// @brief: A class behaves as a lock in a scope. When trying to enter into the
/// critical section, creat a object of this class. After the control path goes
/// out of the scope, it will release the lock automatically.
template <class LockType> class ScopedAcquire {
public:
/// @brief: When constructing, acquire the lock.
/// @param: lock(Input), pointer to an existing lock.
explicit ScopedAcquire(LockType* lock) : lock_(lock), doRelease(true) {
static_assert(isMutex<LockType>::value, "ScopedAcquire requires a mutex type.");
lock_.Acquire();
}
explicit ScopedAcquire(LockType lock) : lock_(lock), doRelease(true) {
static_assert(!isMutex<LockType>::value, "Mutex types are not copyable.");
lock_.Acquire();
}
/// @brief: when destructing, release the lock.
~ScopedAcquire() {
if (doRelease) lock_.Release();
}
/// @brief: Release the lock early. Avoid using when possible.
void Release() {
lock_.Release();
doRelease = false;
}
private:
/// @brief: Adapts between pointers to mutex types and mutex pointer types.
template <class T, bool B> class container {
public:
container(T* lock) : lock_(lock) {}
__forceinline bool Acquire() { return lock_->Acquire(); }
__forceinline void Release() { return lock_->Release(); }
private:
T* lock_;
};
/// @brief: Specialization for mutex pointer types.
template <class T> class container<T, false> {
public:
container(T lock) : lock_(lock) {}
__forceinline bool Acquire() { return lock_.Acquire(); }
__forceinline void Release() { return lock_.Release(); }
private:
T lock_;
};
container<LockType, isMutex<LockType>::value> lock_;
bool doRelease;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(ScopedAcquire);
};
} // namespace rocr
#endif // HSA_RUNTIME_CORE_SUTIL_LOCKS_H_
util/os.h
+327
Wyświetl plik
@@ -0,0 +1,327 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
// Minimal operating system abstraction interfaces.
#ifndef HSA_RUNTIME_CORE_UTIL_OS_H_
#define HSA_RUNTIME_CORE_UTIL_OS_H_
#include <string>
#include <vector>
#include "utils.h"
namespace rocr {
namespace os {
typedef void* LibHandle;
typedef void* Semaphore;
typedef void* Mutex;
typedef void* SharedMutex;
typedef void* Thread;
typedef void* EventHandle;
enum class os_t { OS_WIN = 0, OS_LINUX, COUNT };
static __forceinline std::underlying_type<os_t>::type os_index(os_t val) {
return std::underlying_type<os_t>::type(val);
}
#ifdef _WIN32
static const os_t current_os = os_t::OS_WIN;
#elif __linux__
static const os_t current_os = os_t::OS_LINUX;
#else
static_assert(false, "Operating System not detected!");
#endif
/// @brief: Loads dynamic library based on file name. Return value will be NULL
/// if failed.
/// @param: filename(Input), file name of the library.
/// @return: LibHandle.
LibHandle LoadLib(std::string filename);
/// @brief: Gets the address of exported symbol. Return NULl if failed.
/// @param: lib(Input), library handle which exporting from.
/// @param: export_name(Input), the name of the exported symbol.
/// @return: void*.
void* GetExportAddress(LibHandle lib, std::string export_name);
/// @brief: Unloads the dynamic library.
/// @param: lib(Input), library handle which will be unloaded.
void CloseLib(LibHandle lib);
/// @brief: Lists loaded tool libraries that contain
/// symbol HSA_AMD_TOOL_PRIORITY
/// @return: List of library handles
std::vector<LibHandle> GetLoadedToolsLib();
/// @brief: Returns the library's path name.
/// @param: lib(Input), libray handle
/// @return: Path name of library
std::string GetLibraryName(LibHandle lib);
/// @brief: Creates a Semaphore, will return NULL if failed.
/// @param: void.
/// @return: Semaphore.
Semaphore CreateSemaphore();
/// @brief: Waits for the semaphore. This is a blocking wait.
/// If the Semaphore is signalled, this function will return.
/// @param: sem(Input), handle to the semaphore.
/// @return: void.
bool WaitSemaphore(Semaphore sem);
/// @brief: Post/Signal/Wake-up the semaphore
/// @param: sem(Input), handle to the semaphore.
/// @return: void.
void PostSemaphore(Semaphore sem);
/// @brief: Destroys the semaphore.
/// @param: sem(Input), handle to the semaphore.
/// @return: void.
void DestroySemaphore(Semaphore sem);
/// @brief: Creates a mutex, will return NULL if failed.
/// @param: void.
/// @return: Mutex.
Mutex CreateMutex();
/// @brief: Tries to acquire the mutex once, if successed, return true.
/// @param: lock(Input), handle to the mutex.
/// @return: bool.
bool TryAcquireMutex(Mutex lock);
/// @brief: Aquires the mutex, if the mutex is locked, it will wait until it is
/// released. If the mutex is acquired successfully, it will return true.
/// @param: lock(Input), handle to the mutex.
/// @return: bool.
bool AcquireMutex(Mutex lock);
/// @brief: Releases the mutex.
/// @param: lock(Input), handle to the mutex.
/// @return: void.
void ReleaseMutex(Mutex lock);
/// @brief: Destroys the mutex.
/// @param: lock(Input), handle to the mutex.
/// @return: void.
void DestroyMutex(Mutex lock);
/// @brief: Creates a shared mutex, will return NULL if failed.
/// @param: void.
/// @return: SharedMutex.
SharedMutex CreateSharedMutex();
/// @brief: Tries to acquire the mutex in exclusive mode once, if successed, return true.
/// @param: lock(Input), handle to the shared mutex.
/// @return: bool.
bool TryAcquireSharedMutex(SharedMutex lock);
/// @brief: Aquires the mutex in exclusive mode, if the mutex is locked, it will wait until it is
/// released. If the mutex is acquired successfully, it will return true.
/// @param: lock(Input), handle to the mutex.
/// @return: bool.
bool AcquireSharedMutex(SharedMutex lock);
/// @brief: Releases the mutex from exclusive mode.
/// @param: lock(Input), handle to the mutex.
/// @return: void.
void ReleaseSharedMutex(SharedMutex lock);
/// @brief: Tries to acquire the mutex in shared mode once, if successed, return true.
/// @param: lock(Input), handle to the mutex.
/// @return: bool.
bool TrySharedAcquireSharedMutex(SharedMutex lock);
/// @brief: Aquires the mutex in shared mode, if the mutex in exclusive mode, it will wait until it
/// is released. If the mutex is acquired successfully, it will return true.
/// @param: lock(Input), handle to the mutex.
/// @return: bool.
bool SharedAcquireSharedMutex(SharedMutex lock);
/// @brief: Releases the mutex from shared mode.
/// @param: lock(Input), handle to the mutex.
/// @return: void.
void SharedReleaseSharedMutex(SharedMutex lock);
/// @brief: Destroys the mutex.
/// @param: lock(Input), handle to the mutex.
/// @return: void.
void DestroySharedMutex(SharedMutex lock);
/// @brief: Puts current thread to sleep.
/// @param: delayInMs(Input), time in millisecond for sleeping.
/// @return: void.
void Sleep(int delayInMs);
/// @brief: Puts current thread to sleep.
/// @param: delayInMs(Input), time in millisecond for sleeping.
/// @return: void.
void uSleep(int delayInUs);
/// @brief: Yields current thread.
/// @param: void.
/// @return: void.
void YieldThread();
typedef void (*ThreadEntry)(void*);
/// @brief: Creates a thread will return NULL if failed.
/// @param: entry_function(Input), a pointer to the function which the thread
/// starts from.
/// @param: entry_argument(Input), a pointer to the argument of the thread
/// function.
/// @param: stack_size(Input), size of the thread's stack, 0 by default.
/// @return: Thread, a handle to thread created.
Thread CreateThread(ThreadEntry entry_function, void* entry_argument,
uint stack_size = 0);
/// @brief: Destroys the thread.
/// @param: thread(Input), thread handle to what will be destroyed.
/// @return: void.
void CloseThread(Thread thread);
/// @brief: Waits for specific thread to finish, if successful, return true.
/// @param: thread(Input), handle to waiting thread.
/// @return: bool.
bool WaitForThread(Thread thread);
/// @brief: Waits for multiple threads to finish, if successful, return true.
/// @param; threads(Input), a pointer to a list of thread handle.
/// @param: thread_count(Input), number of threads to be waited on.
/// @return: bool.
bool WaitForAllThreads(Thread* threads, uint thread_count);
/// @brief: Determines if environment key is set.
/// @param: env_var_name(Input), name of the environment value.
/// @return: bool, true for binding any value to environment key,
/// including an empty string. False otherwise
bool IsEnvVarSet(std::string env_var_name);
/// @brief: Sets the environment value.
/// @param: env_var_name(Input), name of the environment value.
/// @param: env_var_value(Input), value of the environment value.s
/// @return: void.
void SetEnvVar(std::string env_var_name, std::string env_var_value);
/// @brief: Gets the value of environment value.
/// @param: env_var_name(Input), name of the environment value.
/// @return: std::string, value of the environment value, returned as string.
std::string GetEnvVar(std::string env_var_name);
/// @brief: Gets the process ID.
/// @param: void
/// @return: int, process ID returned as int.
int GetProcessId();
/// @brief: Gets the max virtual memory size accessible to the application.
/// @param: void.
/// @return: size_t, size of the accessible memory to the application.
size_t GetUserModeVirtualMemorySize();
/// @brief: Gets the max physical host system memory size.
/// @param: void.
/// @return: size_t, size of the physical host system memory.
size_t GetUsablePhysicalHostMemorySize();
/// @brief: Gets the virtual memory base address. It is hardcoded to 0.
/// @param: void.
/// @return: uintptr_t, always 0.
uintptr_t GetUserModeVirtualMemoryBase();
/// @brief os event api, create an event
/// @param: auto_reset whether an event can reset the status automatically
/// @param: init_state initial state of the event
/// @return: event handle
EventHandle CreateOsEvent(bool auto_reset, bool init_state);
/// @brief os event api, destroy an event
/// @param: event handle
/// @return: whether destroy is correct
int DestroyOsEvent(EventHandle event);
/// @brief os event api, wait on event
/// @param: event Event handle
/// @param: milli_seconds wait time
/// @return: Indicate success or timeout
int WaitForOsEvent(EventHandle event, unsigned int milli_seconds);
/// @brief os event api, set event state
/// @param: event Event handle
/// @return: Whether event set is correct
int SetOsEvent(EventHandle event);
/// @brief os event api, reset event state
/// @param: event Event handle
/// @return: Whether event reset is correct
int ResetOsEvent(EventHandle event);
/// @brief reads a clock which is deemed to be accurate for elapsed time
/// measurements, though not necessarilly fast to query
/// @return clock counter value
uint64_t ReadAccurateClock();
/// @brief retrieves the frequency in Hz of the unit used in ReadAccurateClock.
/// It does not necessarilly reflect the resolution of the clock, but is the
/// value needed to convert a difference in the clock's counter value to elapsed
/// seconds. This frequency does not change at runtime.
/// @return returns the frequency
uint64_t AccurateClockFrequency();
/// @brief read the system clock which serves as the HSA system clock
/// counter in KFD.
uint64_t ReadSystemClock();
/// @brief read the system clock frequency
uint64_t SystemClockFrequency();
typedef struct cpuid_s {
char ManufacturerID[13]; // 12 char, NULL terminated
bool mwaitx;
} cpuid_t;
/// @brief parse CPUID
/// @param: cpuinfo struct to be filled
bool ParseCpuID(cpuid_t* cpuinfo);
} // namespace os
} // namespace rocr
#endif // HSA_RUNTIME_CORE_UTIL_OS_H_
util/simple_heap.h
+363
Wyświetl plik
@@ -0,0 +1,363 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
// A simple best fit memory allocator with eager compaction. Manages block sub-allocation.
// For use when memory efficiency is more important than allocation speed.
// O(log n) time.
#ifndef HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_
#define HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_
#include <map>
#include <deque>
#include <utility>
#include "core/util/utils.h"
namespace rocr {
template <typename Allocator> class SimpleHeap {
private:
struct Fragment_T {
typedef std::multimap<size_t, uintptr_t>::iterator ptr_t;
ptr_t free_list_entry_;
struct {
size_t size : 62;
bool discard : 1;
bool free : 1;
};
Fragment_T(ptr_t Iterator, size_t Len, bool Free)
: free_list_entry_(Iterator), size(Len), discard(false), free(Free) {}
Fragment_T() = default;
};
struct Block {
uintptr_t base_ptr_;
size_t length_;
Block(uintptr_t base, size_t length) : base_ptr_(base), length_(length) {}
Block() = default;
};
Allocator block_allocator_;
std::multimap<size_t, uintptr_t> free_list_;
std::map<uintptr_t, std::map<uintptr_t, Fragment_T>> block_list_;
std::deque<Block> block_cache_;
// Size of blocks that are at least partially in use.
size_t in_use_size_;
// Total size of block cache
size_t cache_size_;
__forceinline bool isFree(const Fragment_T& node) { return node.free; }
__forceinline void setUsed(Fragment_T& node) {
node.free = false;
node.free_list_entry_ = free_list_.end();
}
__forceinline void setFree(Fragment_T& node, typename Fragment_T::ptr_t Iterator) {
node.free_list_entry_ = Iterator;
node.free = true;
}
__forceinline Fragment_T makeFragment(size_t Len) {
return Fragment_T(free_list_.end(), Len, false);
}
__forceinline Fragment_T makeFragment(typename Fragment_T::ptr_t Iterator, size_t Len) {
return Fragment_T(Iterator, Len, true);
}
__forceinline void removeFreeListEntry(Fragment_T& node) {
if (node.free_list_entry_ != free_list_.end()) {
free_list_.erase(node.free_list_entry_);
node.free_list_entry_ = free_list_.end();
}
}
__forceinline void discard(Fragment_T& node) {
removeFreeListEntry(node);
node.discard = true;
}
public:
explicit SimpleHeap(const Allocator& BlockAllocator = Allocator())
: block_allocator_(BlockAllocator), in_use_size_(0), cache_size_(0) {}
~SimpleHeap() {
trim();
// Leak here may be due to the user. Check is for debugging only.
// assert(in_use_size_ == 0 && "Leak in SimpleHeap.");
}
SimpleHeap(const SimpleHeap& rhs) = delete;
SimpleHeap(SimpleHeap&& rhs) = delete;
SimpleHeap& operator=(const SimpleHeap& rhs) = delete;
SimpleHeap& operator=(SimpleHeap&& rhs) = delete;
void* alloc(size_t bytes) {
// Find best fit.
uintptr_t base;
size_t size;
// For bytes >= 2MB, the requested mem should be aligned
size_t align_bytes = bytes;
const int retry = bytes >= GPU_HUGE_PAGE_SIZE ? 1 : 0;
size_t align = bytes >= GPU_HUGE_PAGE_SIZE ? GPU_HUGE_PAGE_SIZE : DEFAULT_GPU_PAGE_SIZE;
for (int i = 0; i <= retry; i++) {
auto free_fragment = free_list_.lower_bound(align_bytes);
if (free_fragment == free_list_.end()) break;
uintptr_t addr = free_fragment->second;
size = free_fragment->first;
assert(size >= bytes && "SimpleHeap: map lower_bound failure.");
// Find the containing block and fragment
auto it = block_list_.upper_bound(addr);
it--;
auto& frag_map = it->second;
const auto& fragment = frag_map.find(addr);
assert(fragment != frag_map.end() && "Inconsistency in SimpleHeap.");
assert(size == fragment->second.size && "Inconsistency in SimpleHeap.");
size_t delta = addr & (align - 1);
if (!delta) {
// already find aligned address
base = addr;
free_list_.erase(free_fragment);
// Sub-allocate from fragment.
fragment->second.size = bytes;
setUsed(fragment->second);
// Record remaining free space.
if (size > bytes) {
free_fragment = free_list_.insert(std::make_pair(size - bytes, base + bytes));
frag_map[base + bytes] = makeFragment(free_fragment, size - bytes);
}
} else {
// If this is the first request and the requested size is not enough for alignment,
// then request for a bigger hole and do trim.
if (i == 0 && size < bytes + align - delta) {
align_bytes += align;
continue;
}
uintptr_t aligned_base = addr + align - delta;
base = aligned_base;
// Erase the old free list
free_list_.erase(free_fragment);
// fragment 1 - free
free_fragment = free_list_.insert(std::make_pair(aligned_base - addr, addr));
frag_map[addr] = makeFragment(free_fragment, aligned_base - addr);
//fragment 2 - used
frag_map[base] = makeFragment(bytes);
// fragement 3 - free
if (size > aligned_base - addr + bytes) {
free_fragment = free_list_.insert(std::make_pair(size - (aligned_base - addr) - bytes, aligned_base + bytes));
frag_map[aligned_base + bytes] = makeFragment(free_fragment, size - (aligned_base - addr) - bytes);
}
}
return reinterpret_cast<void*>(base);
}
// No usable fragment, check block cache
if (bytes < default_block_size() && !block_cache_.empty()) {
const auto& block = block_cache_.back();
base = block.base_ptr_;
size = block.length_;
block_cache_.pop_back();
cache_size_ -= size;
} else { // Alloc new block - new block may be larger than default.
void* ptr = block_allocator_.alloc(bytes, size);
base = reinterpret_cast<uintptr_t>(ptr);
assert(ptr != nullptr && "Block allocation failed, Allocator is expected to throw.");
}
in_use_size_ += size;
assert(size >= bytes && "Alloc exceeds block size.");
// Sub alloc and insert free region.
if (size > bytes) {
auto free_fragment = free_list_.insert(std::make_pair(size - bytes, base + bytes));
block_list_[base][base + bytes] = makeFragment(free_fragment, size - bytes);
}
// Track used region
block_list_[base][base] = makeFragment(bytes);
// Disallow multiple suballocation from large blocks.
// Prevents a small allocation from retaining a large block.
if (bytes > default_block_size()) {
bool err = discardBlock(reinterpret_cast<void*>(base));
assert(err && "Large block discard failed.");
}
return reinterpret_cast<void*>(base);
}
bool free(void* ptr) {
if (ptr == nullptr) return true;
uintptr_t base = reinterpret_cast<uintptr_t>(ptr);
// Find fragment and validate.
auto frag_map_it = block_list_.upper_bound(base);
if (frag_map_it == block_list_.begin()) return false;
frag_map_it--;
auto& frag_map = frag_map_it->second;
auto fragment = frag_map.find(base);
if (fragment == frag_map.end() || isFree(fragment->second)) return false;
bool discard = fragment->second.discard;
// Merge lower
if (fragment != frag_map.begin()) {
auto lower = fragment;
lower--;
if (isFree(lower->second)) {
removeFreeListEntry(lower->second);
lower->second.size += fragment->second.size;
frag_map.erase(fragment);
fragment = lower;
}
}
// Merge upper
{
auto upper = fragment;
upper++;
if ((upper != frag_map.end()) && isFree(upper->second)) {
removeFreeListEntry(upper->second);
fragment->second.size += upper->second.size;
frag_map.erase(upper);
}
}
// Release whole free blocks.
if (frag_map.size() == 1) {
Block block(fragment->first, fragment->second.size);
block_list_.erase(frag_map_it);
// Discard or add to the block cache.
if (discard) {
block_allocator_.free(reinterpret_cast<void*>(block.base_ptr_), block.length_);
} else {
block_cache_.push_back(block);
cache_size_ += block.length_;
in_use_size_ -= block.length_;
}
balance();
// Don't publish free space since block was moved to the cache.
return true;
}
// Don't report free memory if discarding the fragment.
if (discard) return true;
// Report free fragment
const auto& freeEntry =
free_list_.insert(std::make_pair(size_t(fragment->second.size), fragment->first));
setFree(fragment->second, freeEntry);
return true;
}
void balance() {
// Release old blocks when over cache limit.
while ((block_cache_.size() > 1) && (cache_size_ > in_use_size_ * 2)) {
const auto& block = block_cache_.front();
block_allocator_.free(reinterpret_cast<void*>(block.base_ptr_), block.length_);
cache_size_ -= block.length_;
block_cache_.pop_front();
}
}
void trim() {
for (const auto& block : block_cache_)
block_allocator_.free(reinterpret_cast<void*>(block.base_ptr_), block.length_);
block_cache_.clear();
cache_size_ = 0;
}
size_t cache_size() const { return cache_size_; }
size_t default_block_size() const { return block_allocator_.block_size(); }
// Prevent reuse of the block containing ptr. No further fragments will be allocated from the
// block and the block will not be added to the block cache when it is free.
bool discardBlock(void* ptr) {
if (ptr == nullptr) return true;
uintptr_t base = reinterpret_cast<uintptr_t>(ptr);
// Find block validate.
auto frag_map_it = block_list_.upper_bound(base);
if (frag_map_it == block_list_.begin()) return false;
frag_map_it--;
auto& frag_map = frag_map_it->second;
if ((base < frag_map.begin()->first) ||
(frag_map.rbegin()->first + frag_map.rbegin()->second.size <= base))
return false;
// Is block already discarded?
if (frag_map.begin()->second.discard) return true;
// Mark all fragments for discard and compute block size. Removes freelist records for all
// fragments in the block.
size_t size = 0;
for (auto& frag : frag_map) {
discard(frag.second);
size += frag.second.size;
}
// Remove discarded block from in-use tracking and rebalance the block cache.
in_use_size_ -= size;
balance();
return true;
}
};
} // namespace rocr
#endif // HSA_RUNTME_CORE_UTIL_SIMPLE_HEAP_H_
util/small_heap.cpp
+185
Wyświetl plik
@@ -0,0 +1,185 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#include "small_heap.h"
namespace rocr {
// Inserts node into freelist after place.
// Assumes node will not be an end of the list (list has guard nodes).
void SmallHeap::insertafter(SmallHeap::iterator_t place, SmallHeap::iterator_t node) {
assert(place->first < node->first && "Order violation");
assert(isfree(place->second) && "Freelist operation error.");
iterator_t next = place->second.next;
node->second.next = next;
node->second.prior = place;
place->second.next = node;
next->second.prior = node;
}
// Removes node from freelist.
// Assumes node will not be an end of the list (list has guard nodes).
void SmallHeap::remove(SmallHeap::iterator_t node) {
assert(isfree(node->second) && "Freelist operation error.");
node->second.prior->second.next = node->second.next;
node->second.next->second.prior = node->second.prior;
setused(node->second);
}
// Returns high if merge failed or the merged node.
SmallHeap::memory_t::iterator SmallHeap::merge(SmallHeap::memory_t::iterator low,
SmallHeap::memory_t::iterator high) {
assert(isfree(low->second) && "Merge with allocated block");
assert(isfree(high->second) && "Merge with allocated block");
if ((char*)low->first + low->second.len != (char*)high->first) return high;
assert(!islastfree(high->second) && "Illegal merge.");
low->second.len += high->second.len;
low->second.next = high->second.next;
high->second.next->second.prior = low;
memory.erase(high);
return low;
}
void SmallHeap::free(void* ptr) {
if (ptr == nullptr) return;
auto iterator = memory.find(ptr);
// Check for illegal free
if (iterator == memory.end()) {
assert(false && "Illegal free.");
return;
}
// Return memory to total and link node into free list
total_free += iterator->second.len;
// Could also traverse the free list which might be faster in some cases.
auto before = iterator;
before--;
while (!isfree(before->second)) before--;
assert(before->second.next->first > iterator->first && "Inconsistency in small heap.");
insertafter(before, iterator);
// Attempt compaction
iterator = merge(before, iterator);
merge(iterator, iterator->second.next);
// Update lowHighBondary
high.erase(ptr);
}
void* SmallHeap::alloc(size_t bytes) {
// Is enough memory available?
if ((bytes > total_free) || (bytes == 0)) return nullptr;
iterator_t current;
// Walk the free list and allocate at first fitting location
current = firstfree();
while (!islastfree(current->second)) {
if (bytes <= current->second.len) {
// Decrement from total
total_free -= bytes;
// Split node
if (bytes != current->second.len) {
void* remaining = (char*)current->first + bytes;
Node& node = memory[remaining];
node.len = current->second.len - bytes;
current->second.len = bytes;
insertafter(current, memory.find(remaining));
}
remove(current);
return current->first;
}
current = current->second.next;
}
assert(current->second.len == 0 && "Freelist corruption.");
// Can't service the request due to fragmentation
return nullptr;
}
void* SmallHeap::alloc_high(size_t bytes) {
// Is enough memory available?
if ((bytes > total_free) || (bytes == 0)) return nullptr;
iterator_t current;
// Walk the free list and allocate at first fitting location
current = lastfree();
while (!isfirstfree(current->second)) {
if (bytes <= current->second.len) {
// Decrement from total
total_free -= bytes;
void* alloc;
// Split node
if (bytes != current->second.len) {
alloc = (char*)current->first + current->second.len - bytes;
current->second.len -= bytes;
Node& node = memory[alloc];
node.len = bytes;
setused(node);
} else {
alloc = current->first;
remove(current);
}
high.insert(alloc);
return alloc;
}
current = current->second.prior;
}
assert(current->second.len == 0 && "Freelist corruption.");
// Can't service the request due to fragmentation
return nullptr;
}
} // namespace rocr
util/small_heap.h
+131
Wyświetl plik
@@ -0,0 +1,131 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
// A simple first fit memory allocator with eager compaction. For use with few
// items (where list iteration is faster than trees).
// Not thread safe!
#ifndef HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_
#define HSA_RUNTME_CORE_UTIL_SMALL_HEAP_H_
#include <map>
#include <set>
#include "utils.h"
namespace rocr {
class SmallHeap {
private:
struct Node;
typedef std::map<void*, Node> memory_t;
typedef memory_t::iterator iterator_t;
struct Node {
size_t len;
iterator_t next;
iterator_t prior;
};
SmallHeap(const SmallHeap& rhs) = delete;
SmallHeap& operator=(const SmallHeap& rhs) = delete;
void* const pool;
const size_t length;
size_t total_free;
memory_t memory;
std::set<void*> high;
__forceinline bool isfree(const Node& node) const { return node.next != memory.begin(); }
__forceinline bool islastfree(const Node& node) const { return node.next == memory.end(); }
__forceinline bool isfirstfree(const Node& node) const { return node.prior == memory.end(); }
__forceinline void setlastfree(Node& node) { node.next = memory.end(); }
__forceinline void setfirstfree(Node& node) { node.prior = memory.end(); }
__forceinline void setused(Node& node) { node.next = memory.begin(); }
__forceinline iterator_t firstfree() { return memory.begin()->second.next; }
__forceinline iterator_t lastfree() { return memory.rbegin()->second.prior; }
void insertafter(iterator_t place, iterator_t node);
void remove(iterator_t node);
iterator_t merge(iterator_t low, iterator_t high);
public:
SmallHeap() : pool(nullptr), length(0), total_free(0) {}
SmallHeap(void* base, size_t length)
: pool(base), length(length), total_free(length) {
assert(pool != nullptr && "Invalid base address.");
assert(pool != (void*)0xFFFFFFFFFFFFFFFFull && "Invalid base address.");
assert((char*)pool + length != (char*)0xFFFFFFFFFFFFFFFFull && "Invalid pool bounds.");
Node& start = memory[0];
Node& node = memory[pool];
Node& end = memory[(void*)0xFFFFFFFFFFFFFFFFull];
start.len = 0;
start.next = memory.find(pool);
setfirstfree(start);
node.len = length;
node.prior = memory.begin();
node.next = --memory.end();
end.len = 0;
end.prior = start.next;
setlastfree(end);
high.insert((void*)0xFFFFFFFFFFFFFFFFull);
}
void* alloc(size_t bytes);
void* alloc_high(size_t bytes);
void free(void* ptr);
void* base() const { return pool; }
size_t size() const { return length; }
size_t remaining() const { return total_free; }
void* high_split() const { return *high.begin(); }
};
} // namespace rocr
#endif
util/timer.cpp
+111
Wyświetl plik
@@ -0,0 +1,111 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#include "core/util/timer.h"
namespace rocr {
namespace timer {
accurate_clock::init::init() {
freq = os::AccurateClockFrequency();
accurate_clock::period_ns = 1e9 / double(freq);
}
// Calibrates the fast clock using the accurate clock.
fast_clock::init::init() {
typedef accurate_clock clock;
clock::duration delay(std::chrono::milliseconds(1));
// calibrate clock
fast_clock::raw_rep min = 0;
clock::duration elapsed;
do {
elapsed = clock::duration::max();
for (int t = 0; t < 10; t++) {
fast_clock::raw_rep r1, r2;
clock::time_point t0, t1, t2, t3;
t0 = clock::now();
std::atomic_signal_fence(std::memory_order_acq_rel);
r1 = fast_clock::raw_now();
std::atomic_signal_fence(std::memory_order_acq_rel);
t1 = clock::now();
std::atomic_signal_fence(std::memory_order_acq_rel);
do {
t2 = clock::now();
} while (t2 - t1 < delay);
std::atomic_signal_fence(std::memory_order_acq_rel);
r2 = fast_clock::raw_now();
std::atomic_signal_fence(std::memory_order_acq_rel);
t3 = clock::now();
// If elapsed time is shorter than last recorded time and both the start
// and end times are confirmed correlated then record the clock readings.
// This protects against inaccuracy due to thread switching
if ((t3 - t1 < elapsed) && ((t1 - t0) * 10 < (t2 - t1)) &&
((t3 - t2) * 10 < (t2 - t1))) {
elapsed = t3 - t1;
min = r2 - r1;
}
}
delay += delay;
} while (min < 1000);
fast_clock::freq = double(min) / duration_in_seconds(elapsed);
fast_clock::period_ps = 1e12 / fast_clock::freq;
// printf("Timer setup took %f ms\n", duration_in_seconds(elapsed)*1000.0f);
// printf("Fast clock frequency: %f MHz\n", double(fast_clock::freq)/1e6);
}
double accurate_clock::period_ns;
accurate_clock::raw_frequency accurate_clock::freq;
accurate_clock::init accurate_clock::accurate_clock_init;
double fast_clock::period_ps;
fast_clock::raw_frequency fast_clock::freq;
fast_clock::init fast_clock::fast_clock_init;
} // namespace timer
} // namespace rocr
util/timer.h
+173
Wyświetl plik
@@ -0,0 +1,173 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef HSA_RUNTIME_CORE_UTIL_TIMER_H_
#define HSA_RUNTIME_CORE_UTIL_TIMER_H_
#include "core/util/utils.h"
#include "core/util/os.h"
#include <chrono>
#include <time.h>
#include <type_traits>
namespace rocr {
namespace timer {
// Needed to patch around a mixed arithmetic bug in MSVC's duration_cast as of
// VS 2013.
template <bool isFloat, bool isSigned>
struct wide_type {
typedef double type;
};
template <>
struct wide_type<false, false> {
typedef uintmax_t type;
};
template <>
struct wide_type<false, true> {
typedef intmax_t type;
};
template <typename To, typename Rep, typename Period>
static __forceinline To
duration_cast(const std::chrono::duration<Rep, Period>& d) {
typedef typename wide_type<std::is_floating_point<Rep>::value,
std::is_signed<Rep>::value>::type wide;
typedef std::chrono::duration<wide, typename To::period> unit_convert_t;
unit_convert_t temp = std::chrono::duration_cast<unit_convert_t>(d);
return To(static_cast<typename To::rep>(temp.count()));
}
// End patch
template <typename Rep, typename Period>
static __forceinline double duration_in_seconds(
std::chrono::duration<Rep, Period> delta) {
typedef std::chrono::duration<double, std::ratio<1, 1>> seconds;
return seconds(delta).count();
}
template <typename rep>
static __forceinline rep duration_from_seconds(double delta) {
typedef std::chrono::duration<double, std::ratio<1, 1>> seconds;
return std::chrono::duration_cast<rep>(seconds(delta));
}
// Provices a C++11 standard clock interface to the os::AccurateClock functions
class accurate_clock {
public:
typedef double rep;
typedef std::nano period;
typedef std::chrono::duration<rep, period> duration;
typedef std::chrono::time_point<accurate_clock> time_point;
static const bool is_steady = true;
static __forceinline time_point now() {
return time_point(duration(raw_now() * period_ns));
}
// These two extra APIs and types let us use clocks without conversion to the
// arbitrary period unit
typedef uint64_t raw_rep;
typedef uint64_t raw_frequency;
static __forceinline raw_rep raw_now() { return os::ReadAccurateClock(); }
static __forceinline raw_frequency raw_freq() { return freq; }
private:
static double period_ns;
static raw_frequency freq;
class init {
public:
init();
};
static init accurate_clock_init;
};
// Provices a C++11 standard clock interface to the lowest latency approximate
// clock
class fast_clock {
public:
typedef double rep;
typedef std::pico period;
typedef std::chrono::duration<rep, period> duration;
typedef std::chrono::time_point<fast_clock> time_point;
static const bool is_steady = true;
static __forceinline time_point now() {
return time_point(duration(raw_now() * period_ps));
}
// These two extra APIs and types let us use clocks without conversion to the
// arbitrary period unit
typedef uint64_t raw_rep;
typedef double raw_frequency;
#if defined(__x86_64__) || defined(_M_X64)
static __forceinline raw_rep raw_now() { return __rdtsc(); }
static __forceinline raw_frequency raw_freq() { return freq; }
#else
static __forceinline raw_rep raw_now() {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
return (raw_rep(ts.tv_sec) * 1000000000 + raw_rep(ts.tv_nsec));
}
static __forceinline raw_frequency raw_freq() { return 1.e-9; }
#endif
private:
static double period_ps;
static raw_frequency freq;
class init {
public:
init();
};
static init fast_clock_init;
};
} // namespace timer
} // namespace rocr
#endif
util/utils.h
+424
Wyświetl plik
@@ -0,0 +1,424 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2024, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
// Generally useful utility functions
#ifndef HSA_RUNTIME_CORE_UTIL_UTILS_H_
#define HSA_RUNTIME_CORE_UTIL_UTILS_H_
#include "stdint.h"
#include "stddef.h"
#include "stdlib.h"
#include "stdarg.h"
#include "unistd.h"
#include <assert.h>
#include <iostream>
#include <string>
#include <algorithm>
#include <sstream>
#include <thread>
namespace rocr {
extern FILE* log_file;
extern uint8_t log_flags[8];
typedef unsigned int uint;
typedef uint64_t uint64;
#if defined(__GNUC__)
#if defined(__i386__) || defined(__x86_64__)
#include <x86intrin.h>
#endif
// 2MB huge page size
#define GPU_HUGE_PAGE_SIZE (2 << 20)
// 4KB page size
#define DEFAULT_GPU_PAGE_SIZE (1 << 12)
#define __forceinline __inline__ __attribute__((always_inline))
#define __declspec(x) __attribute__((x))
#undef __stdcall
#define __stdcall // __attribute__((__stdcall__))
#define __ALIGNED__(x) __attribute__((aligned(x)))
void log_printf(const char* file, int line, const char* format, ...);
static __forceinline void* _aligned_malloc(size_t size, size_t alignment) {
#ifdef _ISOC11_SOURCE
return aligned_alloc(alignment, size);
#else
void *mem = NULL;
if (0 != posix_memalign(&mem, alignment, size)) return NULL;
return mem;
#endif
}
static __forceinline void _aligned_free(void* ptr) { return free(ptr); }
#elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))
#include "intrin.h"
#define __ALIGNED__(x) __declspec(align(x))
#if (_MSC_VER < 1800) // < VS 2013
static __forceinline unsigned long long int strtoull(const char* str,
char** endptr, int base) {
return static_cast<unsigned long long>(_strtoui64(str, endptr, base));
}
#endif
#if (_MSC_VER < 1900) // < VS 2015
#define thread_local __declspec(thread)
#endif
#else
#error "Compiler and/or processor not identified."
#endif
#define STRING2(x) #x
#define STRING(x) STRING2(x)
#define PASTE2(x, y) x##y
#define PASTE(x, y) PASTE2(x, y)
#ifdef NDEBUG
#define debug_warning_n(exp, limit) \
do { \
} while (false)
#else
#define debug_warning_n(exp, limit) \
do { \
static std::atomic<int> count(0); \
if (!(exp) && (limit == 0 || count < limit)) { \
fprintf(stderr, "Warning: " STRING(exp) " in %s, " __FILE__ ":" STRING(__LINE__) "\n", \
__PRETTY_FUNCTION__); \
count++; \
} \
} while (false)
#endif
#define debug_warning(exp) debug_warning_n((exp), 0)
#ifdef NDEBUG
#define debug_print(fmt, ...) \
do { \
} while (false)
#else
#define debug_print(fmt, ...) \
do { \
fprintf(stderr, fmt, ##__VA_ARGS__); \
} while (false)
#endif
#ifdef NDEBUG
#define ifdebug if (false)
#else
#define ifdebug if (true)
#endif
#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
#define LogPrint(flag, format, ...) \
do { \
if (hsa_flag_isset64(log_flags, flag)) \
rocr::log_printf(__FILENAME__, __LINE__, format, ##__VA_ARGS__); \
} while (false);
// A macro to disallow the copy and move constructor and operator= functions
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName&) = delete; \
TypeName(TypeName&&) = delete; \
void operator=(const TypeName&) = delete; \
void operator=(TypeName&&) = delete;
template <typename lambda>
class ScopeGuard {
public:
explicit __forceinline ScopeGuard(const lambda& release)
: release_(release), dismiss_(false) {}
ScopeGuard(ScopeGuard& rhs) { *this = rhs; }
__forceinline ~ScopeGuard() {
if (!dismiss_) release_();
}
__forceinline ScopeGuard& operator=(ScopeGuard& rhs) {
dismiss_ = rhs.dismiss_;
release_ = rhs.release_;
rhs.dismiss_ = true;
return *this;
}
__forceinline void Dismiss() { dismiss_ = true; }
private:
lambda release_;
bool dismiss_;
};
template <typename lambda>
static __forceinline ScopeGuard<lambda> MakeScopeGuard(lambda rel) {
return ScopeGuard<lambda>(rel);
}
#define MAKE_SCOPE_GUARD_HELPER(lname, sname, ...) \
auto lname = __VA_ARGS__; \
ScopeGuard<decltype(lname)> sname(lname);
#define MAKE_SCOPE_GUARD(...) \
MAKE_SCOPE_GUARD_HELPER(PASTE(scopeGuardLambda, __COUNTER__), \
PASTE(scopeGuard, __COUNTER__), __VA_ARGS__)
#define MAKE_NAMED_SCOPE_GUARD(name, ...) \
MAKE_SCOPE_GUARD_HELPER(PASTE(scopeGuardLambda, __COUNTER__), name, \
__VA_ARGS__)
/// @brief: Finds out the min one of two inputs, input must support ">"
/// operator.
/// @param: a(Input), a reference to type T.
/// @param: b(Input), a reference to type T.
/// @return: T.
template <class T>
static __forceinline T Min(const T& a, const T& b) {
return (a > b) ? b : a;
}
template <class T, class... Arg>
static __forceinline T Min(const T& a, const T& b, Arg... args) {
return Min(a, Min(b, args...));
}
/// @brief: Find out the max one of two inputs, input must support ">" operator.
/// @param: a(Input), a reference to type T.
/// @param: b(Input), a reference to type T.
/// @return: T.
template <class T>
static __forceinline T Max(const T& a, const T& b) {
return (b > a) ? b : a;
}
template <class T, class... Arg>
static __forceinline T Max(const T& a, const T& b, Arg... args) {
return Max(a, Max(b, args...));
}
/// @brief: Free the memory space which is newed previously.
/// @param: ptr(Input), a pointer to memory space. Can't be NULL.
/// @return: void.
struct DeleteObject {
template <typename T>
void operator()(const T* ptr) const {
delete ptr;
}
};
/// @brief: Checks if a value is power of two, if it is, return true. Be careful
/// when passing 0.
/// @param: val(Input), the data to be checked.
/// @return: bool.
template <typename T>
static __forceinline bool IsPowerOfTwo(T val) {
return (val & (val - 1)) == 0;
}
/// @brief: Calculates the floor value aligned based on parameter of alignment.
/// If value is at the boundary of alignment, it is unchanged.
/// @param: value(Input), value to be calculated.
/// @param: alignment(Input), alignment value.
/// @return: T.
template <typename T>
static __forceinline T AlignDown(T value, size_t alignment) {
return (T)((value / alignment) * alignment);
}
/// @brief: Same as previous one, but first parameter becomes pointer, for more
/// info, see the previous desciption.
/// @param: value(Input), pointer to type T.
/// @param: alignment(Input), alignment value.
/// @return: T*, pointer to type T.
template <typename T>
static __forceinline T* AlignDown(T* value, size_t alignment) {
return (T*)AlignDown((intptr_t)value, alignment);
}
/// @brief: Calculates the ceiling value aligned based on parameter of
/// alignment.
/// If value is at the boundary of alignment, it is unchanged.
/// @param: value(Input), value to be calculated.
/// @param: alignment(Input), alignment value.
/// @param: T.
template <typename T>
static __forceinline T AlignUp(T value, size_t alignment) {
return AlignDown((T)(value + alignment - 1), alignment);
}
/// @brief: Same as previous one, but first parameter becomes pointer, for more
/// info, see the previous desciption.
/// @param: value(Input), pointer to type T.
/// @param: alignment(Input), alignment value.
/// @return: T*, pointer to type T.
template <typename T>
static __forceinline T* AlignUp(T* value, size_t alignment) {
return (T*)AlignDown((intptr_t)((uint8_t*)value + alignment - 1), alignment);
}
/// @brief: Checks if the input value is at the boundary of alignment, if it is,
/// @return true.
/// @param: value(Input), value to be checked.
/// @param: alignment(Input), alignment value.
/// @return: bool.
template <typename T>
static __forceinline bool IsMultipleOf(T value, size_t alignment) {
return (AlignUp(value, alignment) == value);
}
/// @brief: Same as previous one, but first parameter becomes pointer, for more
/// info, see the previous desciption.
/// @param: value(Input), pointer to type T.
/// @param: alignment(Input), alignment value.
/// @return: bool.
template <typename T>
static __forceinline bool IsMultipleOf(T* value, size_t alignment) {
return (AlignUp(value, alignment) == value);
}
static __forceinline uint32_t NextPow2(uint32_t value) {
if (value == 0) return 1;
uint32_t v = value - 1;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
return v + 1;
}
static __forceinline uint64_t NextPow2(uint64_t value) {
if (value == 0) return 1;
uint64_t v = value - 1;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
v |= v >> 32;
return v + 1;
}
static __forceinline bool strIsEmpty(const char* str) noexcept { return str[0] == '\0'; }
static __forceinline std::string& ltrim(std::string& s) {
auto it = std::find_if(s.begin(), s.end(),
[](char c) { return !std::isspace<char>(c, std::locale::classic()); });
s.erase(s.begin(), it);
return s;
}
static __forceinline std::string& rtrim(std::string& s) {
auto it = std::find_if(s.rbegin(), s.rend(),
[](char c) { return !std::isspace<char>(c, std::locale::classic()); });
s.erase(it.base(), s.end());
return s;
}
static __forceinline std::string& trim(std::string& s) { return ltrim(rtrim(s)); }
} // namespace rocr
template <uint32_t lowBit, uint32_t highBit, typename T>
static __forceinline uint32_t BitSelect(T p) {
static_assert(sizeof(T) <= sizeof(uintptr_t), "Type out of range.");
static_assert(highBit < sizeof(uintptr_t) * 8, "Bit index out of range.");
uintptr_t ptr = p;
if (highBit != (sizeof(uintptr_t) * 8 - 1))
return (uint32_t)((ptr & ((1ull << (highBit + 1)) - 1)) >> lowBit);
else
return (uint32_t)(ptr >> lowBit);
}
inline uint32_t PtrLow16Shift8(const void* p) {
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
return (uint32_t)((ptr & 0xFFFFULL) >> 8);
}
inline uint32_t PtrHigh64Shift16(const void* p) {
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
return (uint32_t)((ptr & 0xFFFFFFFFFFFF0000ULL) >> 16);
}
inline uint32_t PtrLow40Shift8(const void* p) {
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
return (uint32_t)((ptr & 0xFFFFFFFFFFULL) >> 8);
}
inline uint32_t PtrHigh64Shift40(const void* p) {
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
return (uint32_t)((ptr & 0xFFFFFF0000000000ULL) >> 40);
}
static inline uint8_t Ptr48High8(const void* p) {
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
return (uint8_t)((ptr & 0xFF0000000000ULL) >> 40);
}
static inline uint32_t Ptr48Low32(const void* p) {
uintptr_t ptr = reinterpret_cast<uintptr_t>(p);
assert((ptr & 0xFFFFFFFFFF00ULL) == ptr);
return (uint32_t)((ptr & 0xFFFFFFFFFFULL) >> 8);
}
inline uint32_t PtrLow32(const void* p) {
return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
}
inline uint32_t PtrHigh32(const void* p) {
uint32_t ptr = 0;
#ifdef HSA_LARGE_MODEL
ptr = static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p) >> 32);
#endif
return ptr;
}
inline uint32_t HighPart(uint64_t value) {
return (value & 0xFFFFFFFF00000000) >> 32;
}
inline uint32_t LowPart(uint64_t value) {
return (value & 0x00000000FFFFFFFF);
}
#include "atomic_helpers.h"
#endif // HSA_RUNTIME_CORE_UTIL_UTILS_H_
util/win/os_win.cpp
+327
Wyświetl plik
@@ -0,0 +1,327 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2020, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#ifdef _WIN32 // Are we compiling for windows?
#define NOMINMAX
#include "core/util/os.h"
#include <algorithm>
#include <process.h>
#include <string>
#include <windows.h>
#include <emmintrin.h>
#include <pmmintrin.h>
#include <xmmintrin.h>
#undef Yield
#undef CreateMutex
namespace rocr {
namespace os {
static_assert(sizeof(LibHandle) == sizeof(HMODULE),
"OS abstraction size mismatch");
static_assert(sizeof(LibHandle) == sizeof(::HANDLE),
"OS abstraction size mismatch");
static_assert(sizeof(Semaphore) == sizeof(::HANDLE),
"OS abstraction size mismatch");
static_assert(sizeof(Mutex) == sizeof(::HANDLE),
"OS abstraction size mismatch");
static_assert(sizeof(Thread) == sizeof(::HANDLE),
"OS abstraction size mismatch");
static_assert(sizeof(EventHandle) == sizeof(::HANDLE),
"OS abstraction size mismatch");
LibHandle LoadLib(std::string filename) {
HMODULE ret = LoadLibrary(filename.c_str());
return *(LibHandle*)&ret;
}
void* GetExportAddress(LibHandle lib, std::string export_name) {
return GetProcAddress(*(HMODULE*)&lib, export_name.c_str());
}
void CloseLib(LibHandle lib) { FreeLibrary(*(::HMODULE*)&lib); }
std::vector<LibHandle> GetLoadedLibs() {
// Use EnumProcessModulesEx
static_assert(false, "Not implemented.");
}
std::string GetLibraryName(LibHandle lib) {
static_assert(false, "Not implemented.");
}
Semaphore CreateSemaphore() {
sem = static_cast<void*>(CreateSemaphore(NULL, 0, LONG_MAX, NULL));
assert(sem != NULL && "CreateSemaphore failed");
return *(Semaphore*)&sem;
}
bool WaitSemaphore(Semaphore sem) {
return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0;
}
void PostSemaphore(Semaphore sem) {
ReleaseSemaphore(static_cast<HANDLE>(*sem), 1, NULL);
}
void DestroySemaphore(Semaphore sem) {
if (!CloseHandle(static_cast<HANDLE>(*sem))) {
assert("CloseHandle() failed");
}
*sem = NULL;
}
Mutex CreateMutex() { return CreateEvent(NULL, false, true, NULL); }
bool TryAcquireMutex(Mutex lock) {
return WaitForSingleObject(*(::HANDLE*)&lock, 0) == WAIT_OBJECT_0;
}
bool AcquireMutex(Mutex lock) {
return WaitForSingleObject(*(::HANDLE*)&lock, INFINITE) == WAIT_OBJECT_0;
}
void ReleaseMutex(Mutex lock) { SetEvent(*(::HANDLE*)&lock); }
void DestroyMutex(Mutex lock) { CloseHandle(*(::HANDLE*)&lock); }
void Sleep(int delay_in_millisecond) { ::Sleep(delay_in_millisecond); }
void uSleep(int delayInUs) { ::Sleep(delayInUs / 1000); }
void YieldThread() { ::Sleep(0); }
struct ThreadArgs {
void* entry_args;
ThreadEntry entry_function;
};
unsigned __stdcall ThreadTrampoline(void* arg) {
ThreadArgs* thread_args = (ThreadArgs*)arg;
ThreadEntry entry = thread_args->entry_function;
void* data = thread_args->entry_args;
delete thread_args;
entry(data);
_endthreadex(0);
return 0;
}
Thread CreateThread(ThreadEntry entry_function, void* entry_argument,
uint stack_size) {
ThreadArgs* thread_args = new ThreadArgs();
thread_args->entry_args = entry_argument;
thread_args->entry_function = entry_function;
uintptr_t ret =
_beginthreadex(NULL, stack_size, ThreadTrampoline, thread_args, 0, NULL);
return *(Thread*)&ret;
}
void CloseThread(Thread thread) { CloseHandle(*(::HANDLE*)&thread); }
bool WaitForThread(Thread thread) {
return WaitForSingleObject(*(::HANDLE*)&thread, INFINITE) == WAIT_OBJECT_0;
}
bool WaitForAllThreads(Thread* threads, uint thread_count) {
return WaitForMultipleObjects(thread_count, threads, TRUE, INFINITE) ==
WAIT_OBJECT_0;
}
void SetEnvVar(std::string env_var_name, std::string env_var_value) {
SetEnvironmentVariable(env_var_name.c_str(), env_var_value.c_str());
}
std::string GetEnvVar(std::string env_var_name) {
char* buff;
DWORD char_count = GetEnvironmentVariable(env_var_name.c_str(), NULL, 0);
if (char_count == 0) return "";
buff = (char*)alloca(sizeof(char) * char_count);
GetEnvironmentVariable(env_var_name.c_str(), buff, char_count);
buff[char_count - 1] = '\0';
std::string ret = buff;
return ret;
}
size_t GetUserModeVirtualMemorySize() {
SYSTEM_INFO system_info = {0};
GetSystemInfo(&system_info);
return ((size_t)system_info.lpMaximumApplicationAddress + 1);
}
size_t GetUsablePhysicalHostMemorySize() {
MEMORYSTATUSEX memory_status = {0};
memory_status.dwLength = sizeof(memory_status);
if (GlobalMemoryStatusEx(&memory_status) == 0) {
return 0;
}
const size_t physical_size = static_cast<size_t>(memory_status.ullTotalPhys);
return std::min(GetUserModeVirtualMemorySize(), physical_size);
}
uintptr_t GetUserModeVirtualMemoryBase() { return (uintptr_t)0; }
// Os event wrappers
EventHandle CreateOsEvent(bool auto_reset, bool init_state) {
EventHandle evt = reinterpret_cast<EventHandle>(
CreateEvent(NULL, (BOOL)(!auto_reset), (BOOL)init_state, NULL));
return evt;
}
int DestroyOsEvent(EventHandle event) {
if (event == NULL) {
return -1;
}
return CloseHandle(reinterpret_cast<::HANDLE>(event));
}
int WaitForOsEvent(EventHandle event, unsigned int milli_seconds) {
if (event == NULL) {
return -1;
}
int ret_code =
WaitForSingleObject(reinterpret_cast<::HANDLE>(event), milli_seconds);
if (ret_code == WAIT_TIMEOUT) {
ret_code = 0x14003; // 0x14003 indicates timeout
}
return ret_code;
}
int SetOsEvent(EventHandle event) {
if (event == NULL) {
return -1;
}
return SetEvent(reinterpret_cast<::HANDLE>(event));
}
int ResetOsEvent(EventHandle event) {
if (event == NULL) {
return -1;
}
return ResetEvent(reinterpret_cast<::HANDLE>(event));
}
uint64_t ReadAccurateClock() {
uint64_t ret;
QueryPerformanceCounter((LARGE_INTEGER*)&ret);
return ret;
}
uint64_t AccurateClockFrequency() {
uint64_t ret;
QueryPerformanceFrequency((LARGE_INTEGER*)&ret);
return ret;
}
SharedMutex CreateSharedMutex() {
assert(false && "Not implemented.");
abort();
return nullptr;
}
bool TryAcquireSharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
return false;
}
bool AcquireSharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
return false;
}
void ReleaseSharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
}
bool TrySharedAcquireSharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
return false;
}
bool SharedAcquireSharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
return false;
}
void SharedReleaseSharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
}
void DestroySharedMutex(SharedMutex lock) {
assert(false && "Not implemented.");
abort();
}
uint64_t ReadSystemClock() {
assert(false && "Not implemented.");
abort();
return 0;
}
uint64_t SystemClockFrequency() {
assert(false && "Not implemented.");
abort();
return 0;
}
bool ParseCpuID(cpuid_t* cpuinfo) {
assert(false && "Not implemented.");
abort();
return false;
}
} // namespace os
} // namespace rocr
#endif