diff --git a/rocclr/device/rocm/CMakeLists.txt b/rocclr/device/rocm/CMakeLists.txt index 30cc867cb5..41529d7dd1 100644 --- a/rocclr/device/rocm/CMakeLists.txt +++ b/rocclr/device/rocm/CMakeLists.txt @@ -20,6 +20,8 @@ add_library(oclrocm OBJECT rocappprofile.cpp rocsettings.cpp rocschedcl.cpp + rochcmessages.cpp + rochcprintf.cpp rochostcall.cpp ) set_target_properties(oclrocm PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/rocclr/device/rocm/rochcmessages.cpp b/rocclr/device/rocm/rochcmessages.cpp new file mode 100644 index 0000000000..c65f0bb8b8 --- /dev/null +++ b/rocclr/device/rocm/rochcmessages.cpp @@ -0,0 +1,178 @@ +/* Copyright (c) 2020-present Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ + +#include "utils/debug.hpp" +#include "top.hpp" +#include "utils/flags.hpp" + +#include "rochcmessages.hpp" + +#include + +enum { + DESCRIPTOR_OFFSET_FLAG_BEGIN = 0, + DESCRIPTOR_OFFSET_FLAG_END = 1, + DESCRIPTOR_OFFSET_RESERVED0 = 2, + DESCRIPTOR_OFFSET_LEN = 5, + DESCRIPTOR_OFFSET_ID = 8 +}; + +enum { + DESCRIPTOR_WIDTH_FLAG_BEGIN = 1, + DESCRIPTOR_WIDTH_FLAG_END = 1, + DESCRIPTOR_WIDTH_RESERVED0 = 3, + DESCRIPTOR_WIDTH_LEN = 3, + DESCRIPTOR_WIDTH_ID = 56 +}; + +struct Message { + std::vector data_; + bool live_; + uint64_t messageId_; + + void clear() { + live_ = false; + data_.clear(); + }; + + void append(uint64_t* payload, uint8_t len) { data_.insert(data_.end(), payload, payload + len); } + + Message(uint64_t c) : live_(true), messageId_(c){}; +}; + +static uint64_t getField(uint64_t desc, uint8_t offset, uint8_t width) { + return (desc >> offset) & ((1UL << width) - 1); +} + +static uint64_t setField(uint64_t desc, uint64_t value, uint8_t offset, uint8_t width) { + uint64_t resetMask = ~(((1UL << width) - 1) << offset); + return (desc & resetMask) | (value << offset); +} + +static uint64_t getLen(uint64_t desc) { + return getField(desc, DESCRIPTOR_OFFSET_LEN, DESCRIPTOR_WIDTH_LEN); +} + +static uint64_t getBeginFlag(uint64_t desc) { + return getField(desc, DESCRIPTOR_OFFSET_FLAG_BEGIN, DESCRIPTOR_WIDTH_FLAG_BEGIN); +} + +static uint64_t getEndFlag(uint64_t desc) { + return getField(desc, DESCRIPTOR_OFFSET_FLAG_END, DESCRIPTOR_WIDTH_FLAG_END); +} + +static uint64_t resetBeginFlag(uint64_t desc) { + uint64_t resetMask = ~(1 << DESCRIPTOR_OFFSET_FLAG_BEGIN); + return desc & resetMask; +} + +static uint64_t getMessageId(uint64_t desc) { + return getField(desc, DESCRIPTOR_OFFSET_ID, DESCRIPTOR_WIDTH_ID); +} + +static uint64_t setMessageId(uint64_t desc, uint64_t id) { + return setField(desc, id, DESCRIPTOR_OFFSET_ID, DESCRIPTOR_WIDTH_ID); +} + +Message* MessageHandler::newMessage() { + if (!freeSlots_.empty()) { + auto c = freeSlots_.back(); + freeSlots_.pop_back(); + assert(c <= messageSlots_.size()); + Message* m = messageSlots_[c]; + assert(m); + assert(m->messageId_ == c); + assert(m->data_.empty()); + m->live_ = true; + return m; + } + + Message* m = new Message(messageSlots_.size()); + messageSlots_.push_back(m); + return m; +} + +MessageHandler::~MessageHandler() { + for (auto M: messageSlots_) { + delete M; + } +} + +Message* MessageHandler::getMessage(uint64_t messageId) { + if (messageSlots_.size() <= messageId) { + return nullptr; + } + + auto m = messageSlots_[messageId]; + + return m->live_ ? m : nullptr; +} + +void MessageHandler::discardMessage(Message* message) { + message->clear(); + freeSlots_.push_back(message->messageId_); + // TODO: Consider reducing the number of message slots based on + // some busy-ness heuristic. +} + +// Defined in rochcprintf.cpp +void handlePrintf(uint64_t* output, const uint64_t* input, uint64_t len); + +bool MessageHandler::handlePayload(uint32_t service, uint64_t* payload) { + Message* message = nullptr; + + auto desc = payload[0]; + auto begin = getBeginFlag(desc); + auto end = getEndFlag(desc); + + if (begin) { + message = newMessage(); + desc = resetBeginFlag(desc); + desc = setMessageId(desc, message->messageId_); + payload[0] = desc; + } else { + auto messageId = getMessageId(desc); + message = getMessage(messageId); + } + + if (!message) { + ClPrint(amd::LOG_ERROR, amd::LOG_ALWAYS, "Hostcall: No message found"); + return false; + } + + auto len = getLen(desc); + message->append(payload + 1, len); + + if (!end) { + return true; + } + + switch (service) { + case SERVICE_PRINTF: + handlePrintf(payload, message->data_.data(), message->data_.size()); + break; + default: + ClPrint(amd::LOG_ERROR, amd::LOG_ALWAYS, "Hostcall: Messages not supported for service %d", + service); + return false; + } + discardMessage(message); + return true; +} diff --git a/rocclr/device/rocm/rochcmessages.hpp b/rocclr/device/rocm/rochcmessages.hpp new file mode 100644 index 0000000000..164c6cb08b --- /dev/null +++ b/rocclr/device/rocm/rochcmessages.hpp @@ -0,0 +1,88 @@ +/* Copyright (c) 2020-present Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ + +/** \file Concatenating hostcalls into a message + * + * A message is a stream of 64-bit integers transmitted as a series + * of hostcall invocations by the device code. Although the hostcall + * is "warp-wide", the message for each workitem is distinct. + * + * Of the eight uint64_t arguments in hostcall, the first argument is + * used as the message descriptor, while the rest are used for + * message contents. The descriptor consists of the following fields: + * + * - Bit 0 is the BEGIN flag. + * - Bit 1 is the END flag. + * - Bits 2-4 are reserved and must be zero. + * - Bits 5-7 indicate the number of elements being transmitted. + * - Bits 8-63 contain a 56-bit message ID. + * + * A hostcall with the BEGIN flag set in the descriptor indicates the + * start of a new message. A hostcall with the END flag set indicates + * the end of a message. A single hostcall can have both flags set if + * the message fits in the payload of a single hostcall. Each + * hostcall indicates the number of uint64_t elements in the payload + * that contain data to be appended to the message. + * + * When the message receives a hostcall with the BEGIN flag set, it allocates a + * new message ID, which is transmitted to the device via the first return + * value in the hostcall. Every subsequent hostcall containing the same message + * ID appends its payload to that message. The message is said to be "active" + * until a corresponding END hostcall is received. + * + * When the message handler receives a hostcall with the END flag set, it + * invokes the corresponding message handler on the contents of the + * accumulated message, and then discards the message. The handler + * may return up to two uint64_t values, that are transmitted to the + * device via the return value of the last hostcall. + * + * Behaviour is undefined in each of the following cases: + * - An END packet is received with a non-existent message ID, or with + * the ID of an inactive message (previously terminated by an END packet). + * - No END packet is received for an active message. + * - Any of the reserved bits are non-zero. + * - Different hostcalls indicate the same active message ID but a + * different service. + */ + +#pragma once + +#include + +enum ServiceID { + SERVICE_RESERVED = 0, + SERVICE_FUNCTION_CALL = 1, + SERVICE_PRINTF = 2, +}; + +struct Message; + +class MessageHandler { + std::vector freeSlots_; + std::vector messageSlots_; + + Message* newMessage(); + Message* getMessage(uint64_t desc); + void discardMessage(Message* message); + + public: + ~MessageHandler(); + bool handlePayload(uint32_t service, uint64_t* payload); +}; diff --git a/rocclr/device/rocm/rochcprintf.cpp b/rocclr/device/rocm/rochcprintf.cpp new file mode 100644 index 0000000000..21b3687c35 --- /dev/null +++ b/rocclr/device/rocm/rochcprintf.cpp @@ -0,0 +1,232 @@ +/* Copyright (c) 2020-present Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ + +/** \file Format string processing for printf based on hostcall messages. + */ + +#include +#include +#include +#include +#include + +static void checkPrintf(int* outCount, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + int retval = vprintf(fmt, args); + *outCount = retval < 0 ? retval : *outCount + retval; +} + +static int countStars(const std::string& spec) { + int stars = 0; + for (auto c : spec) { + if (c == '*') { + ++stars; + } + } + return stars; +} + +template +static const uint64_t* consumeInteger(int* outCount, const std::string& spec, const uint64_t* ptr, + Args... args) { + checkPrintf(outCount, spec.c_str(), args..., ptr[0]); + return ptr + 1; +} + +template +static const uint64_t* consumeFloatingPoint(int* outCount, const std::string& spec, + const uint64_t* ptr, Args... args) { + double d; + memcpy(&d, ptr, 8); + checkPrintf(outCount, spec.c_str(), args..., d); + return ptr + 1; +} + +template +static const uint64_t* consumeCstring(int* outCount, const std::string& spec, const uint64_t* ptr, + Args... args) { + auto str = reinterpret_cast(ptr); + checkPrintf(outCount, spec.c_str(), args..., str); + return ptr + (strlen(str) + 7) / 8; +} + +template +static const uint64_t* consumePointer(int* outCount, const std::string& spec, const uint64_t* ptr, + Args... args) { + auto vptr = reinterpret_cast(*ptr); + checkPrintf(outCount, spec.c_str(), args..., vptr); + return ptr + 1; +} + +template +static const uint64_t* consumeArgument(int* outCount, const std::string& spec, const uint64_t* ptr, + const uint64_t* end, Args... args) { + switch (spec.back()) { + case 'd': + case 'i': + case 'o': + case 'u': + case 'x': + case 'X': + case 'c': + return consumeInteger(outCount, spec, ptr, args...); + case 'f': + case 'F': + case 'e': + case 'E': + case 'g': + case 'G': + case 'a': + case 'A': + return consumeFloatingPoint(outCount, spec, ptr, args...); + case 's': + return consumeCstring(outCount, spec, ptr, args...); + case 'p': + return consumePointer(outCount, spec, ptr, args...); + case 'n': + return ptr + 1; + } + + // Undefined behaviour with an unknown flag + return end; +} + +static const uint64_t* processSpec(int* outCount, const std::string& spec, const uint64_t* ptr, + const uint64_t* end) { + auto stars = countStars(spec); + assert(stars < 3 && "cannot have more than two placeholders"); + switch (stars) { + case 0: + return consumeArgument(outCount, spec, ptr, end); + case 1: + // Undefined behaviour if there are not enough arguments. + if (end - ptr < 2) { + return end; + } + return consumeArgument(outCount, spec, ptr + 1, end, ptr[0]); + case 2: + // Undefined behaviour if there are not enough arguments. + if (end - ptr < 3) { + return end; + } + return consumeArgument(outCount, spec, ptr + 2, end, ptr[0], ptr[1]); + } + + // Undefined behaviour if three are more than two stars. + return end; +} + +/** \brief Process a printf message using the system printf function. + * \param begin Start of the uint64_t array containing the message. + * \param end One past the last element in the array. + * \return An integer that satisfies the POSIX return value for printf. + * + * The message has the following format: + * - uint64_t version, required to be zero. + * - Format string padded to an 8 byte boundary. + * - Sequence of arguments + * - Each int/float/pointer argument occupies one uint64_t location. + * - Each string argument is padded to an 8 byte boundary. + * + * The format() function extracts the format string, and then + * extracts further arguments based on the format string. It breaks + * up the format string at the format specifiers and invokes the + * system printf() function multiple times: + * - A format specifier and its corresponding arguments are passed to + * a separate printf() call. + * - Slices between the format specifiers are passed to additional + * printf() calls interleaved with the specifiers. + * + * Limitations: + * - Behaviour is undefined with wide characters and strings. + * - %n specifier is ignored and the corresponding argument is skipped. + */ +static int format(const uint64_t* begin, const uint64_t* end) { + const char convSpecifiers[] = "diouxXfFeEgGaAcspn"; + auto ptr = begin; + + const std::string fmt(reinterpret_cast(ptr)); + ptr += (fmt.length() + 7 + 1) / 8; // the extra '1' is for the null + + int outCount = 0; + size_t point = 0; + while (true) { + // Each segment of the format string delineated by [mark, + // point) is handled seprately. + auto mark = point; + point = fmt.find('%', point); + + // Two different cases where a literal segment is printed out. + // 1. When the point reaches the end of the format string. + // 2. When the point is at the start of a format specifier. + if (point == std::string::npos) { + checkPrintf(&outCount, "%s", &fmt[mark]); + return outCount; + } + checkPrintf(&outCount, "%.*s", (int)(point - mark), &fmt[mark]); + if (outCount < 0) { + return outCount; + } + + mark = point; + ++point; + + // Handle the simplest specifier, '%%'. + if (fmt[point] == '%') { + checkPrintf(&outCount, "%%"); + if (outCount < 0) { + return outCount; + } + ++point; + continue; + } + + // Before processing the specifier, check if we have run out + // of arguments. + if (ptr == end) { + return outCount; + } + + // Undefined behaviour if we don't see a conversion specifier. + point = fmt.find_first_of(convSpecifiers, point); + if (point == std::string::npos) { + return outCount; + } + ++point; + + // [mark,point) now contains a complete specifier. + const std::string spec(fmt, mark, point - mark); + ptr = processSpec(&outCount, spec, ptr, end); + if (outCount < 0) { + return outCount; + } + } +} + +void handlePrintf(uint64_t* output, const uint64_t* input, uint64_t len) { + auto version = *input; + if (version != 0) { + *output = -1; + return; + } + + *output = format(input + 1, input + len); +} diff --git a/rocclr/device/rocm/rochostcall.cpp b/rocclr/device/rocm/rochostcall.cpp index 9ef887e4d0..c8a6493fcb 100644 --- a/rocclr/device/rocm/rochostcall.cpp +++ b/rocclr/device/rocm/rochostcall.cpp @@ -18,7 +18,13 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#include "utils/debug.hpp" #include "top.hpp" +#include "utils/flags.hpp" + +#include "rochostcall.hpp" +#include "rochcmessages.hpp" + #include "os/os.hpp" #include "thread/monitor.hpp" #include "utils/util.hpp" @@ -35,11 +41,6 @@ namespace { // anonymous -enum ServiceID { - SERVICE_RESERVED = 0, - SERVICE_FUNCTION_CALL, -}; - enum SignalValue { SIGNAL_DONE = 0, SIGNAL_INIT = 1 }; /** \brief Packet payload @@ -111,7 +112,7 @@ class HostcallBuffer { Payload* getPayload(uint64_t ptr) const; public: - void processPackets(); + void processPackets(MessageHandler& messages); void initialize(uint32_t num_packets); void setDoorbell(hsa_signal_t doorbell) { doorbell_ = doorbell; }; }; @@ -150,30 +151,31 @@ static uint32_t resetReadyFlag(uint32_t control) { */ typedef void (*HostcallFunctionCall)(uint64_t* output, const uint64_t* input); -static void handleFunctionCall(void* state, uint32_t service, uint64_t* payload) { - uint64_t output[2]; - - auto fptr = reinterpret_cast(payload[0]); - fptr(output, payload + 1); - memcpy(payload, output, sizeof(output)); -} - -static bool handlePayload(uint32_t service, uint64_t* payload) { +static void handlePayload(MessageHandler& messages, uint32_t service, uint64_t* payload) { switch (service) { - case SERVICE_FUNCTION_CALL: - handleFunctionCall(nullptr, service, payload); - return true; - break; + case SERVICE_FUNCTION_CALL: { + uint64_t output[2]; + auto fptr = reinterpret_cast(payload[0]); + fptr(output, payload + 1); + memcpy(payload, output, sizeof(output)); + return; + } + case SERVICE_PRINTF: + if (!messages.handlePayload(service, payload)) { + ClPrint(amd::LOG_ERROR, amd::LOG_ALWAYS, "Hostcall: invalid request for service \"%d\".", + service); + amd::report_fatal(__FILE__, __LINE__, "Hostcall: invalid service request."); + } + return; default: ClPrint(amd::LOG_ERROR, amd::LOG_ALWAYS, "Hostcall: no handler found for service ID \"%d\".", service); amd::report_fatal(__FILE__, __LINE__, "Hostcall service not supported."); - return false; - break; + return; } } -void HostcallBuffer::processPackets() { +void HostcallBuffer::processPackets(MessageHandler& messages) { // Grab the entire ready stack and set the top to 0. New requests from the // device will continue pushing on the stack while we process the packets that // we have grabbed. @@ -198,7 +200,7 @@ void HostcallBuffer::processPackets() { auto wi = amd::leastBitSet(activemask); activemask ^= static_cast(1) << wi; auto slot = payload->slots[wi]; - handlePayload(service, slot); + handlePayload(messages, service, slot); } __atomic_store_n(&header->control_, resetReadyFlag(header->control_), @@ -262,6 +264,7 @@ void HostcallBuffer::initialize(uint32_t num_packets) { class HostcallListener { std::set buffers_; hsa_signal_t doorbell_; + MessageHandler messages_; class Thread : public amd::Thread { public: @@ -329,7 +332,7 @@ void HostcallListener::consumePackets() { amd::ScopedLock lock{listenerLock}; for (auto ii : buffers_) { - ii->processPackets(); + ii->processPackets(messages_); } }