From 3c4f97f66c7613b982f9108e4cb46873b6a803e4 Mon Sep 17 00:00:00 2001 From: German Andryeyev Date: Wed, 9 Feb 2022 17:16:39 -0500 Subject: [PATCH] SWDEV-286150 - Remove GSL backend Change-Id: Iba9a997ee7d5ff6ac00d5888ff189a4514958fe9 [ROCm/clr commit: 525a1bbf1a4130e4340eca5b6487c69283d1d590] --- projects/clr/rocclr/cmake/FindAMD_UGL.cmake | 2 +- projects/clr/rocclr/device/device.cpp | 174 +- projects/clr/rocclr/device/device.hpp | 9 +- .../clr/rocclr/device/gpu/gpuappprofile.cpp | 40 - .../clr/rocclr/device/gpu/gpuappprofile.hpp | 57 - projects/clr/rocclr/device/gpu/gpubinary.cpp | 513 --- projects/clr/rocclr/device/gpu/gpubinary.hpp | 114 - projects/clr/rocclr/device/gpu/gpublit.cpp | 2458 ------------ projects/clr/rocclr/device/gpu/gpublit.hpp | 427 -- .../clr/rocclr/device/gpu/gpucompiler.cpp | 348 -- .../clr/rocclr/device/gpu/gpuconstbuf.cpp | 98 - .../clr/rocclr/device/gpu/gpuconstbuf.hpp | 83 - .../clr/rocclr/device/gpu/gpucounters.cpp | 97 - .../clr/rocclr/device/gpu/gpucounters.hpp | 140 - .../clr/rocclr/device/gpu/gpudebugger.hpp | 133 - .../clr/rocclr/device/gpu/gpudebugmanager.cpp | 354 -- .../clr/rocclr/device/gpu/gpudebugmanager.hpp | 130 - projects/clr/rocclr/device/gpu/gpudefs.hpp | 354 -- projects/clr/rocclr/device/gpu/gpudevice.cpp | 2263 ----------- projects/clr/rocclr/device/gpu/gpudevice.hpp | 623 --- projects/clr/rocclr/device/gpu/gpukernel.cpp | 3520 ----------------- projects/clr/rocclr/device/gpu/gpukernel.hpp | 877 ---- projects/clr/rocclr/device/gpu/gpumemory.cpp | 1141 ------ projects/clr/rocclr/device/gpu/gpumemory.hpp | 286 -- projects/clr/rocclr/device/gpu/gpuprintf.cpp | 657 --- projects/clr/rocclr/device/gpu/gpuprintf.hpp | 183 - projects/clr/rocclr/device/gpu/gpuprogram.cpp | 1989 ---------- projects/clr/rocclr/device/gpu/gpuprogram.hpp | 535 --- .../clr/rocclr/device/gpu/gpuresource.cpp | 1858 --------- .../clr/rocclr/device/gpu/gpuresource.hpp | 501 --- projects/clr/rocclr/device/gpu/gpusched.hpp | 96 - projects/clr/rocclr/device/gpu/gpuschedcl.cpp | 310 -- projects/clr/rocclr/device/gpu/gpuscsi.cpp | 210 - .../clr/rocclr/device/gpu/gpusettings.cpp | 476 --- .../clr/rocclr/device/gpu/gpusettings.hpp | 135 - .../clr/rocclr/device/gpu/gputhreadtrace.cpp | 77 - .../clr/rocclr/device/gpu/gputhreadtrace.hpp | 148 - .../clr/rocclr/device/gpu/gputimestamp.cpp | 112 - .../clr/rocclr/device/gpu/gputimestamp.hpp | 137 - projects/clr/rocclr/device/gpu/gputrap.hpp | 139 - projects/clr/rocclr/device/gpu/gpuvirtual.cpp | 3333 ---------------- projects/clr/rocclr/device/gpu/gpuvirtual.hpp | 528 --- .../clr/rocclr/device/gpu/gslbe/localdefs | 2 - .../clr/rocclr/device/gpu/gslbe/localrules | 1 - .../gpu/gslbe/src/promotions/Readme.txt | 2 - .../device/gpu/gslbe/src/rt/EventQueue.cpp | 222 -- .../device/gpu/gslbe/src/rt/EventQueue.h | 74 - .../device/gpu/gslbe/src/rt/GSLContext.cpp | 601 --- .../device/gpu/gslbe/src/rt/GSLContext.h | 126 - .../device/gpu/gslbe/src/rt/GSLDevice.cpp | 1662 -------- .../device/gpu/gslbe/src/rt/GSLDevice.h | 258 -- .../gpu/gslbe/src/rt/GSLDeviceD3D10.cpp | 251 -- .../gpu/gslbe/src/rt/GSLDeviceD3D11.cpp | 174 - .../device/gpu/gslbe/src/rt/GSLDeviceD3D9.cpp | 77 - .../device/gpu/gslbe/src/rt/GSLDeviceGL.cpp | 1033 ----- .../device/gpu/gslbe/src/rt/GSLStubs.cpp | 154 - .../device/gpu/gslbe/src/rt/backend.cpp | 194 - .../rocclr/device/gpu/gslbe/src/rt/backend.h | 225 -- .../device/gpu/gslbe/src/rt/caltarget.h | 79 - .../gpu/gslbe/src/rt/inifile/ini_export.cpp | 115 - .../gpu/gslbe/src/rt/inifile/ini_export.h | 33 - .../gpu/gslbe/src/rt/inifile/ini_values.h | 354 -- .../gpu/gslbe/src/rt/inifile/inifile.cpp | 556 --- .../device/gpu/gslbe/src/rt/inifile/inifile.h | 184 - .../gslbe/src/rt/inifile/inifile_parser.cpp | 245 -- .../gpu/gslbe/src/rt/inifile/inifile_parser.h | 61 - .../src/rt => pal}/DxxOpenCLInteropExt.h | 0 projects/clr/rocclr/device/pal/paldefs.hpp | 29 - projects/clr/rocclr/device/pal/paldevice.cpp | 12 +- projects/clr/rocclr/device/pal/paldevice.hpp | 2 +- .../clr/rocclr/device/pal/palresource.cpp | 2 +- .../clr/rocclr/device/rocm/rocsettings.cpp | 4 +- projects/clr/rocclr/utils/flags.cpp | 4 - projects/clr/rocclr/utils/flags.hpp | 2 +- 74 files changed, 92 insertions(+), 32311 deletions(-) delete mode 100644 projects/clr/rocclr/device/gpu/gpuappprofile.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuappprofile.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpubinary.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpubinary.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpublit.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpublit.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpucompiler.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuconstbuf.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuconstbuf.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpucounters.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpucounters.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpudebugger.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpudebugmanager.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpudebugmanager.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpudefs.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpudevice.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpudevice.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpukernel.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpukernel.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpumemory.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpumemory.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuprintf.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuprintf.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuprogram.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuprogram.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuresource.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuresource.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpusched.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuschedcl.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuscsi.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpusettings.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpusettings.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gputhreadtrace.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gputhreadtrace.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gputimestamp.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gputimestamp.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gputrap.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuvirtual.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gpuvirtual.hpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/localdefs delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/localrules delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/promotions/Readme.txt delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D10.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D11.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D9.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceGL.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLStubs.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/caltarget.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_values.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.h delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.cpp delete mode 100644 projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.h rename projects/clr/rocclr/device/{gpu/gslbe/src/rt => pal}/DxxOpenCLInteropExt.h (100%) diff --git a/projects/clr/rocclr/cmake/FindAMD_UGL.cmake b/projects/clr/rocclr/cmake/FindAMD_UGL.cmake index 1246476e9c..0b09ada65b 100644 --- a/projects/clr/rocclr/cmake/FindAMD_UGL.cmake +++ b/projects/clr/rocclr/cmake/FindAMD_UGL.cmake @@ -44,5 +44,5 @@ find_package_handle_standard_args(AMD_UGL AMD_UGL_INCLUDE_DIR) mark_as_advanced(AMD_UGL_INCLUDE_DIR) -set(AMD_UGL_INCLUDE_DIRS ${AMD_UGL_INCLUDE_DIR} ${ROCCLR_SRC_DIR}/device/gpu/gslbe/src/rt) +set(AMD_UGL_INCLUDE_DIRS ${AMD_UGL_INCLUDE_DIR}) mark_as_advanced(AMD_UGL_INCLUDE_DIRS) diff --git a/projects/clr/rocclr/device/device.cpp b/projects/clr/rocclr/device/device.cpp index 218f9e4080..eaa7a97f14 100644 --- a/projects/clr/rocclr/device/device.cpp +++ b/projects/clr/rocclr/device/device.cpp @@ -40,11 +40,6 @@ extern void PalDeviceUnload(); //} #endif // WITH_PAL_DEVICE -#if defined(WITH_GPU_DEVICE) -extern bool DeviceLoad(); -extern void DeviceUnload(); -#endif // WITH_GPU_DEVICE - #include "platform/runtime.hpp" #include "platform/program.hpp" #include "thread/monitor.hpp" @@ -112,86 +107,85 @@ std::pair Isa::supportedIsas() { // If the HSAIL or AMD IL compilers do not support the target, then use // nullptr for the ID. // - // -------------------- Compiler -------------------- ------- Runtime ----- ---- IP ---- --- Target --- ---------- Target Properties ---------- - // Supported Version Features Mem - // SIMD Channel LDS LDS - // SIMD/ SIMD Instr Bank Size/ Mem - // Target ID HSAIL ID ROC PAL GSL Maj/Min/Stp SRAMECC XNACK CU Width Width Width CU Banks - {"gfx700", "Kaveri", true, false, true, 7, 0, 0, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Also Spectre, Spooky, Kalindi - {"gfx701", "Hawaii", true, false, true, 7, 0, 1, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Actually Hawaiipro - {"gfx702", "gfx702", true, false, true, 7, 0, 2, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Actually Hawaii (can execute Hawiipro code) - {"gfx703", nullptr, false, false, true, 7, 0, 3, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Mullins - {"gfx704", "Bonaire", false, false, true, 7, 0, 4, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx705", "Mullins", false, false, true, 7, 0, 5, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Actually Godavari - {"gfx801", nullptr, true, true, true, 8, 0, 1, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx801:xnack-", nullptr, true, false, true, 8, 0, 1, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx801:xnack+", "Carrizo", true, true, true, 8, 0, 1, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx802", "Tonga", true, true, true, 8, 0, 2, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Also Iceland - {"gfx803", "Fiji", true, true, true, 8, 0, 3, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Also Ellesmere/Polaris10, Baffin/Polaris11, Polaris12, Polaris22/VegaM - {"gfx805", nullptr, true, true, true, 8, 0, 5, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Tongapro - {"gfx810", nullptr, true, true, true, 8, 1, 0, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx810:xnack-", nullptr, true, false, true, 8, 1, 0, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx810:xnack+", "Stoney", true, true, true, 8, 1, 0, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx900", "gfx901", true, true, false, 9, 0, 0, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Greenland - {"gfx900:xnack-", "gfx900", true, true, false, 9, 0, 0, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx900:xnack+", "gfx901", true, true, false, 9, 0, 0, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx902", "gfx903", true, true, false, 9, 0, 2, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Raven - {"gfx902:xnack-", "gfx902", true, true, false, 9, 0, 2, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx902:xnack+", "gfx903", true, true, false, 9, 0, 2, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx904", "gfx905", true, true, false, 9, 0, 4, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Vega12 - {"gfx904:xnack-", "gfx904", true, true, false, 9, 0, 4, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx904:xnack+", "gfx905", true, true, false, 9, 0, 4, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906", "gfx907", true, true, false, 9, 0, 6, ANY, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Vega20 - {"gfx906:sramecc-", "gfx907", true, true, false, 9, 0, 6, OFF, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:sramecc+", nullptr, true, true, false, 9, 0, 6, ON, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:xnack-", "gfx906", true, true, false, 9, 0, 6, ANY, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:xnack+", "gfx907", true, true, false, 9, 0, 6, ANY, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:sramecc-:xnack-", "gfx906", true, true, false, 9, 0, 6, OFF, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:sramecc-:xnack+", "gfx907", true, true, false, 9, 0, 6, OFF, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:sramecc+:xnack-", nullptr, true, true, false, 9, 0, 6, ON, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx906:sramecc+:xnack+", nullptr, true, true, false, 9, 0, 6, ON, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908", nullptr, true, false, false, 9, 0, 8, ANY, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:sramecc-", nullptr, true, false, false, 9, 0, 8, OFF, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:sramecc+", nullptr, true, false, false, 9, 0, 8, ON, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:xnack-", nullptr, true, false, false, 9, 0, 8, ANY, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:xnack+", nullptr, true, false, false, 9, 0, 8, ANY, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:sramecc-:xnack-", nullptr, true, false, false, 9, 0, 8, OFF, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:sramecc-:xnack+", nullptr, true, false, false, 9, 0, 8, OFF, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:sramecc+:xnack-", nullptr, true, false, false, 9, 0, 8, ON, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx908:sramecc+:xnack+", nullptr, true, false, false, 9, 0, 8, ON, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx909", nullptr, false, true, false, 9, 0, 2, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Raven2 (can execute Raven code) - {"gfx909:xnack-", nullptr, false, true, false, 9, 0, 2, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx909:xnack+", nullptr, false, true, false, 9, 0, 2, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a", nullptr, true, false, false, 9, 0, 10, ANY, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:sramecc-", nullptr, true, false, false, 9, 0, 10, OFF, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:sramecc+", nullptr, true, false, false, 9, 0, 10, ON, ANY, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:xnack-", nullptr, true, false, false, 9, 0, 10, ANY, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:xnack+", nullptr, true, false, false, 9, 0, 10, ANY, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:sramecc-:xnack-", nullptr, true, false, false, 9, 0, 10, OFF, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:sramecc-:xnack+", nullptr, true, false, false, 9, 0, 10, OFF, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:sramecc+:xnack-", nullptr, true, false, false, 9, 0, 10, ON, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90a:sramecc+:xnack+", nullptr, true, false, false, 9, 0, 10, ON, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90c", nullptr, true, true, false, 9, 0, 12, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Renoir - {"gfx90c:xnack-", "gfx90c", true, true, false, 9, 0, 12, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx90c:xnack+", "gfx90d", true, true, false, 9, 0, 12, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, - {"gfx1010", "gfx1010", true, true, false, 10, 1, 0, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1010:xnack-", "gfx1010", true, true, false, 10, 1, 0, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1010:xnack+", nullptr, true, true, false, 10, 1, 0, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1011", "gfx1011", true, true, false, 10, 1, 1, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1011:xnack-", "gfx1011", true, true, false, 10, 1, 1, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1011:xnack+", nullptr, true, true, false, 10, 1, 1, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1012", "gfx1012", true, true, false, 10, 1, 2, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1012:xnack-", "gfx1012", true, true, false, 10, 1, 2, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1012:xnack+", nullptr, true, true, false, 10, 1, 2, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1013", "gfx1013", true, false, false, 10, 1, 3, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1013:xnack-", "gfx1013", true, false, false, 10, 1, 3, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1013:xnack+", nullptr, true, false, false, 10, 1, 3, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1030", "gfx1030", true, true, false, 10, 3, 0, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1031", "gfx1031", true, true, false, 10, 3, 1, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1032", "gfx1032", true, true, false, 10, 3, 2, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1033", "gfx1033", true, false, false, 10, 3, 3, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1034", "gfx1034", true, true, false, 10, 3, 4, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, - {"gfx1035", "gfx1035", true, true, false, 10, 3, 5, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, + // -------------- Compiler ---------- - Runtime - ---- IP ---- -- Target -- ---------- Target Properties ---------- + // Supported Version Features + // SIMD/ SIMD Instr Bank LDS Mem + // Target ID HSAIL ID ROC PAL Maj/Min/Stp SRAMECC XNACK CU Width Width Width Size Banks + {"gfx700", "Kaveri", true, false, 7, 0, 0, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Also Spectre, Spooky, Kalindi + {"gfx701", "Hawaii", true, false, 7, 0, 1, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Actually Hawaiipro + {"gfx702", "gfx702", true, false, 7, 0, 2, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Actually Hawaii (can execute Hawiipro code) + {"gfx703", nullptr, false, false, 7, 0, 3, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Mullins + {"gfx704", "Bonaire", false, false, 7, 0, 4, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx705", "Mullins", false, false, 7, 0, 5, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Actually Godavari + {"gfx801", nullptr, true, true, 8, 0, 1, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx801:xnack-", nullptr, true, false, 8, 0, 1, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx801:xnack+", "Carrizo", true, true, 8, 0, 1, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx802", "Tonga", true, true, 8, 0, 2, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Also Iceland + {"gfx803", "Fiji", true, true, 8, 0, 3, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Also Ellesmere/Polaris10, Baffin/Polaris11, Polaris12, Polaris22/VegaM + {"gfx805", nullptr, true, true, 8, 0, 5, NONE, NONE, 4, 16, 1, 256, 64 * Ki, 32}, // Tongapro + {"gfx810", nullptr, true, true, 8, 1, 0, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx810:xnack-", nullptr, true, false, 8, 1, 0, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx810:xnack+", "Stoney", true, true, 8, 1, 0, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx900", "gfx901", true, true, 9, 0, 0, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Greenland + {"gfx900:xnack-", "gfx900", true, true, 9, 0, 0, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx900:xnack+", "gfx901", true, true, 9, 0, 0, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx902", "gfx903", true, true, 9, 0, 2, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Raven + {"gfx902:xnack-", "gfx902", true, true, 9, 0, 2, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx902:xnack+", "gfx903", true, true, 9, 0, 2, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx904", "gfx905", true, true, 9, 0, 4, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Vega12 + {"gfx904:xnack-", "gfx904", true, true, 9, 0, 4, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx904:xnack+", "gfx905", true, true, 9, 0, 4, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906", "gfx907", true, true, 9, 0, 6, ANY, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Vega20 + {"gfx906:sramecc-", "gfx907", true, true, 9, 0, 6, OFF, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:sramecc+", nullptr, true, true, 9, 0, 6, ON, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:xnack-", "gfx906", true, true, 9, 0, 6, ANY, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:xnack+", "gfx907", true, true, 9, 0, 6, ANY, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:sramecc-:xnack-", "gfx906", true, true, 9, 0, 6, OFF, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:sramecc-:xnack+", "gfx907", true, true, 9, 0, 6, OFF, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:sramecc+:xnack-", nullptr, true, true, 9, 0, 6, ON, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx906:sramecc+:xnack+", nullptr, true, true, 9, 0, 6, ON, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908", nullptr, true, false, 9, 0, 8, ANY, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:sramecc-", nullptr, true, false, 9, 0, 8, OFF, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:sramecc+", nullptr, true, false, 9, 0, 8, ON, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:xnack-", nullptr, true, false, 9, 0, 8, ANY, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:xnack+", nullptr, true, false, 9, 0, 8, ANY, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:sramecc-:xnack-", nullptr, true, false, 9, 0, 8, OFF, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:sramecc-:xnack+", nullptr, true, false, 9, 0, 8, OFF, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:sramecc+:xnack-", nullptr, true, false, 9, 0, 8, ON, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx908:sramecc+:xnack+", nullptr, true, false, 9, 0, 8, ON, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx909", nullptr, false, true, 9, 0, 2, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Raven2 (can execute Raven code) + {"gfx909:xnack-", nullptr, false, true, 9, 0, 2, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx909:xnack+", nullptr, false, true, 9, 0, 2, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a", nullptr, true, false, 9, 0, 10, ANY, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:sramecc-", nullptr, true, false, 9, 0, 10, OFF, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:sramecc+", nullptr, true, false, 9, 0, 10, ON, ANY, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:xnack-", nullptr, true, false, 9, 0, 10, ANY, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:xnack+", nullptr, true, false, 9, 0, 10, ANY, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:sramecc-:xnack-", nullptr, true, false, 9, 0, 10, OFF, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:sramecc-:xnack+", nullptr, true, false, 9, 0, 10, OFF, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:sramecc+:xnack-", nullptr, true, false, 9, 0, 10, ON, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90a:sramecc+:xnack+", nullptr, true, false, 9, 0, 10, ON, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90c", nullptr, true, true, 9, 0, 12, NONE, ANY, 4, 16, 1, 256, 64 * Ki, 32}, // Also Renoir + {"gfx90c:xnack-", "gfx90c", true, true, 9, 0, 12, NONE, OFF, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx90c:xnack+", "gfx90d", true, true, 9, 0, 12, NONE, ON, 4, 16, 1, 256, 64 * Ki, 32}, + {"gfx1010", "gfx1010", true, true, 10, 1, 0, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1010:xnack-", "gfx1010", true, true, 10, 1, 0, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1010:xnack+", nullptr, true, true, 10, 1, 0, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1011", "gfx1011", true, true, 10, 1, 1, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1011:xnack-", "gfx1011", true, true, 10, 1, 1, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1011:xnack+", nullptr, true, true, 10, 1, 1, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1012", "gfx1012", true, true, 10, 1, 2, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1012:xnack-", "gfx1012", true, true, 10, 1, 2, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1012:xnack+", nullptr, true, true, 10, 1, 2, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1013", "gfx1013", true, false, 10, 1, 3, NONE, ANY, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1013:xnack-", "gfx1013", true, false, 10, 1, 3, NONE, OFF, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1013:xnack+", nullptr, true, false, 10, 1, 3, NONE, ON, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1030", "gfx1030", true, true, 10, 3, 0, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1031", "gfx1031", true, true, 10, 3, 1, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1032", "gfx1032", true, true, 10, 3, 2, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1033", "gfx1033", true, false, 10, 3, 3, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1034", "gfx1034", true, true, 10, 3, 4, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, + {"gfx1035", "gfx1035", true, true, 10, 3, 5, NONE, NONE, 2, 32, 1, 256, 64 * Ki, 32}, }; return std::make_pair(std::begin(supportedIsas_), std::end(supportedIsas_)); } @@ -431,11 +425,6 @@ bool Device::init() { } } #endif // WITH_HSA_DEVICE -#if defined(WITH_GPU_DEVICE) - if (GPU_ENABLE_PAL != 1) { - ret |= DeviceLoad(); - } -#endif // WITH_GPU_DEVICE #if defined(WITH_PAL_DEVICE) if (GPU_ENABLE_PAL != 0) { ret |= PalDeviceLoad(); @@ -455,11 +444,6 @@ void Device::tearDown() { #if defined(WITH_HSA_DEVICE) roc::Device::tearDown(); #endif // WITH_HSA_DEVICE -#if defined(WITH_GPU_DEVICE) - if (GPU_ENABLE_PAL != 1) { - DeviceUnload(); - } -#endif // WITH_GPU_DEVICE #if defined(WITH_PAL_DEVICE) if (GPU_ENABLE_PAL != 0) { PalDeviceUnload(); diff --git a/projects/clr/rocclr/device/device.hpp b/projects/clr/rocclr/device/device.hpp index 5a8e9d0301..ba0182e630 100644 --- a/projects/clr/rocclr/device/device.hpp +++ b/projects/clr/rocclr/device/device.hpp @@ -1351,11 +1351,6 @@ class Isa { return runtimePalSupported_; } - /// @returns If the GSL runtime supports the ISA. - bool runtimeGslSupported() const { - return runtimeGslSupported_; - } - /// @returns SRAM ECC feature status. const Feature &sramecc() const { return sramecc_; @@ -1441,7 +1436,7 @@ class Isa { private: constexpr Isa(const char* targetId, const char* hsailId, - bool runtimeRocSupported, bool runtimePalSupported, bool runtimeGslSupported, + bool runtimeRocSupported, bool runtimePalSupported, uint32_t versionMajor, uint32_t versionMinor, uint32_t versionStepping, Feature sramecc, Feature xnack, uint32_t simdPerCU, uint32_t simdWidth, uint32_t simdInstructionWidth, uint32_t memChannelBankWidth, @@ -1450,7 +1445,6 @@ class Isa { hsailId_(hsailId), runtimeRocSupported_(runtimeRocSupported), runtimePalSupported_(runtimePalSupported), - runtimeGslSupported_(runtimeGslSupported), versionMajor_(versionMajor), versionMinor_(versionMinor), versionStepping_(versionStepping), @@ -1477,7 +1471,6 @@ class Isa { bool runtimeRocSupported_; //!< ROCm runtime is supported. bool runtimePalSupported_; //!< PAL runtime is supported. - bool runtimeGslSupported_; //!< GSL runtime is supported. uint32_t versionMajor_; //!< Isa's major version. uint32_t versionMinor_; //!< Isa's minor version. uint32_t versionStepping_; //!< Isa's stepping version. diff --git a/projects/clr/rocclr/device/gpu/gpuappprofile.cpp b/projects/clr/rocclr/device/gpu/gpuappprofile.cpp deleted file mode 100644 index 417d1a1b26..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuappprofile.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright (c) 2014 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "top.hpp" -#include "utils/debug.hpp" -#include "device/appprofile.hpp" -#include "device/gpu/gpuappprofile.hpp" - -namespace gpu { - -AppProfile::AppProfile() - : amd::AppProfile(), enableHighPerformanceState_(true), reportAsOCL12Device_(false) { - propertyDataMap_.insert({"HighPerfState", PropertyData(DataType_Boolean, &enableHighPerformanceState_)}); - propertyDataMap_.insert({"OCL12Device", PropertyData(DataType_Boolean, &reportAsOCL12Device_)}); - propertyDataMap_.insert({"SclkThreshold", PropertyData(DataType_String, &sclkThreshold_)}); - propertyDataMap_.insert({"DownHysteresis", PropertyData(DataType_String, &downHysteresis_)}); - propertyDataMap_.insert({"UpHysteresis", PropertyData(DataType_String, &upHysteresis_)}); - propertyDataMap_.insert({"PowerLimit", PropertyData(DataType_String, &powerLimit_)}); - propertyDataMap_.insert({"MclkThreshold", PropertyData(DataType_String, &mclkThreshold_)}); - propertyDataMap_.insert({"MclkUpHyst", PropertyData(DataType_String, &mclkUpHyst_)}); - propertyDataMap_.insert({"MclkDownHyst", PropertyData(DataType_String, &mclkDownHyst_)}); -} -} diff --git a/projects/clr/rocclr/device/gpu/gpuappprofile.hpp b/projects/clr/rocclr/device/gpu/gpuappprofile.hpp deleted file mode 100644 index 35d3f79ac1..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuappprofile.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/* Copyright (c) 2014 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUAPPPROFILE_HPP_ -#define GPUAPPPROFILE_HPP_ - -#include -#include - -namespace gpu { - -class AppProfile : public amd::AppProfile { - public: - AppProfile(); - - //! return the value of enableHighPerformanceState_ - bool enableHighPerformanceState() const { return enableHighPerformanceState_; } - bool reportAsOCL12Device() const { return reportAsOCL12Device_; } - const std::string& GetSclkThreshold() const { return sclkThreshold_; } - const std::string& GetDownHysteresis() const { return downHysteresis_; } - const std::string& GetUpHysteresis() const { return upHysteresis_; } - const std::string& GetPowerLimit() const { return powerLimit_; } - const std::string& GetMclkThreshold() const { return mclkThreshold_; } - const std::string& GetMclkUpHyst() const { return mclkUpHyst_; } - const std::string& GetMclkDownHyst() const { return mclkDownHyst_; } - - private: - bool enableHighPerformanceState_; - bool reportAsOCL12Device_; - std::string sclkThreshold_; - std::string downHysteresis_; - std::string upHysteresis_; - std::string powerLimit_; - std::string mclkThreshold_; - std::string mclkUpHyst_; - std::string mclkDownHyst_; -}; -} - -#endif diff --git a/projects/clr/rocclr/device/gpu/gpubinary.cpp b/projects/clr/rocclr/device/gpu/gpubinary.cpp deleted file mode 100644 index 48a3e2b51f..0000000000 --- a/projects/clr/rocclr/device/gpu/gpubinary.cpp +++ /dev/null @@ -1,513 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/gpu/gpubinary.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "utils/options.hpp" -#include "os/os.hpp" -#include -#include - - -namespace { - -enum { NDX_KERNEL = 0, NDX_METADATA = 1, NDX_HEADER = 2, NDX_AMDIL = 3, NDX_LAST }; -typedef struct { - bool IsKernel; // whether the entry is for kernel - - /* - SymInfo[NDX_KERNEL] : SymbolInfo for kernel isa (cal image) - SymInfo[NDX_METADATA] : SymbolInfo for kernel metadata - SymInfo[NDX_HEADER] : SymbolInfo for kernel header - SymInfo[NDX_AMDIL] : SymbolInfo for kernel's amdil - */ - amd::Elf::SymbolInfo SymInfo[NDX_LAST]; -} ElfSymbol_t; -} - -namespace gpu { - -bool ClBinary::loadKernels(NullProgram& program, bool* hasRecompiled) { - const char __OpenCL_[] = "__OpenCL_"; - const char _kernel[] = "_kernel"; - const char _data[] = "_metadata"; // metadata for kernel function - const char _fdata[] = "_fmetadata"; // metadata for non-kernel function - const char _header[] = "_header"; - const char _amdil[] = "_amdil"; - - *hasRecompiled = false; - - // TODO : jugu - // Target should be 15 bit maximum. Should check this somewhere. - uint32_t target = static_cast(dev().calTarget()); - uint16_t elf_target; - amd::Elf::ElfPlatform platform; - if (!elfIn()->getTarget(elf_target, platform)) { - LogError("The OCL binary image loading failed: incorrect format"); - return false; - } - if (platform == amd::Elf::COMPLIB_PLATFORM) { - // BIF 3.0 - uint32_t flag; - aclTargetInfo tgtInfo = amd::Hsail::GetTargetInfo("amdil", nullptr, NULL); - if (!elfIn()->getFlags(flag)) { - LogError("The OCL binary image loading failed: incorrect format"); - return false; - } - if ((elf_target != EM_AMDIL) || (tgtInfo.chip_id != flag)) { - LogError("The OCL binary image loading failed: different target"); - return false; - } - } else { - if (((platform != amd::Elf::CAL_PLATFORM) || ((uint32_t)target != elf_target))) { - LogError("The OCL binary image loading failed: different target"); - return false; - } - } - - /* Using class so that dtor() can be invoked to do clean-up */ - class TempWrapper { - public: - /* - functionNameMap[] maps from a function name (linkage name in the generated code) - to ElfSymbol_t, which is defined as above. - */ - std::unordered_map functionNameMap; - - // Keep all kernel ILs if -use-debugil is present (gpu debugging) - std::unordered_map kernelILs; - - ~TempWrapper() { - for (const auto& it : functionNameMap) { - delete it.second; - } - - kernelILs.clear(); - } - } tempObj; - - /* - If usedebugil is true, we will load IL from .debugil section. We will ignore - _kernel, _amdil, _header in the binary. - */ - bool usedebugil = program.getCompilerOptions()->oVariables->UseDebugIL; - - int num = elfIn()->getSymbolNum(); - for (int ndx = 0; ndx < num; ndx++) { - amd::Elf::SymbolInfo symInfo; - if (!elfIn()->getSymbolInfo(ndx, &symInfo)) { - LogError("LoadKernelFromElf: getSymbolInfo() fails"); - return false; - } - - std::string elfSymName(symInfo.sym_name); - const size_t offset = sizeof(__OpenCL_) - 1; - if (elfSymName.compare(0, offset, __OpenCL_) != 0) { - continue; - } - - // Assume this elfSymName is associated with a kernel name. The following code will adjust - // it if it isn't. - const size_t suffixPos = elfSymName.rfind('_'); - bool isKernel = true; // assume it is a kernel - std::string FName = elfSymName.substr(0, suffixPos); - FName.append("_kernel"); // make the kernel's linkage name - - ElfSymbol_t* elfsymbol = tempObj.functionNameMap[FName]; - amd::Elf::SymbolInfo* sinfo = (elfsymbol != NULL) ? &(elfsymbol->SymInfo[0]) : NULL; - - // Add info for this elf symbol into tempobj's functionNameMap[] - int index = -1; - if (!usedebugil && (elfSymName.compare(suffixPos, sizeof(_kernel) - 1, _kernel) == 0)) { - index = NDX_KERNEL; - assert(((sinfo == NULL) || (sinfo[index].size == 0)) && - "More than one kernel symbol for the same kernel"); - } else if (!usedebugil && (elfSymName.compare(suffixPos, sizeof(_header) - 1, _header) == 0)) { - index = NDX_HEADER; - assert(((sinfo == NULL) || (sinfo[index].size == 0)) && - "More than one header symbol for a kernel"); - } else if (!usedebugil && (elfSymName.compare(suffixPos, sizeof(_amdil) - 1, _amdil) == 0)) { - index = NDX_AMDIL; - assert(((sinfo == NULL) || (sinfo[index].size == 0)) && - "More than one amdil symbol for a kernel"); - } else if (elfSymName.compare(suffixPos, sizeof(_data) - 1, _data) == 0) { - index = NDX_METADATA; - assert(((sinfo == NULL) || (sinfo[index].size == 0)) && - "More than one metadata symbol for the same kernel"); - } else if (elfSymName.compare(suffixPos, sizeof(_fdata) - 1, _fdata) == 0) { - index = NDX_METADATA; - isKernel = false; - - FName = elfSymName.substr(offset, suffixPos - offset); - - elfsymbol = tempObj.functionNameMap[FName]; - sinfo = (elfsymbol != NULL) ? &(elfsymbol->SymInfo[0]) : NULL; - - assert(((sinfo == NULL) || (sinfo[index].size == 0)) && - "More than one metadata symbol for a non-kernel function"); - } - - if (index >= 0) { - if (elfsymbol == NULL) { - elfsymbol = new ElfSymbol_t(); - sinfo = &(elfsymbol->SymInfo[0]); - tempObj.functionNameMap[FName] = elfsymbol; - - elfsymbol->IsKernel = isKernel; - } - sinfo[index] = symInfo; - } - } - - std::string programil; - if (usedebugil) { - char* section; - size_t sz; - - if (elfIn_->getSection(amd::Elf::ILDEBUG, §ion, &sz)) { - // Get debugIL - programil.append(section, sz); - } else { - LogError("LoadKernelFromElf(): reading .debugil failed"); - return false; - } - - // Append all function metadata to debugIL - for (const auto& it : tempObj.functionNameMap) { - ElfSymbol_t* elfsymbol = it.second; - if (elfsymbol == NULL) { - // Not valid, skip - continue; - } - if ((elfsymbol->SymInfo[NDX_METADATA].address != 0) && - (elfsymbol->SymInfo[NDX_METADATA].size > 0)) { - std::string mdString = std::string(elfsymbol->SymInfo[NDX_METADATA].address, - elfsymbol->SymInfo[NDX_METADATA].size); - assert((mdString.find_first_of('\0') == std::string::npos) && - "Metadata string has NULL inside !"); - programil.append(mdString); - } - } - - const char* ilKernelName = program.getCompilerOptions()->oVariables->JustKernel; - if (!program.getAllKernelILs(tempObj.kernelILs, programil, ilKernelName)) { - LogError("LoadKernelFromElf(): MDParser failed generating kernel ILs"); - return false; - } - - // Now, patch the IL from debugIL into functionNameMap[] - for (const auto& it : tempObj.kernelILs) { - const std::string& kn = it.first; - const std::string& ilstr = it.second; - - ElfSymbol_t* elfsymbol = tempObj.functionNameMap[kn]; - if (elfsymbol == NULL) { - elfsymbol = new ElfSymbol_t(); - tempObj.functionNameMap[kn] = elfsymbol; - } - amd::Elf::SymbolInfo* sinfo = &(elfsymbol->SymInfo[0]); - - elfsymbol->IsKernel = true; - sinfo[NDX_AMDIL].address = const_cast(ilstr.data()); - sinfo[NDX_AMDIL].size = ilstr.size(); - // All the other fields in SymInfo is unused - } - } - - bool recompiled = false; - bool hasKernels = false; - for (const auto& it : tempObj.functionNameMap) { - ElfSymbol_t* elfsymbol = it.second; - if (elfsymbol == NULL) { - // Not valid, skip - continue; - } else if (!elfsymbol->IsKernel) { - // Not a kernel. Add its metadata to the OCL binary in case recompilation happens - // and the new binary is needed. - if (false && (elfsymbol->SymInfo[NDX_METADATA].size > 0)) { - std::string fmetadata = "__OpenCL_"; - fmetadata.append(it.first); - fmetadata.append("_fmetadata"); - - if (!elfOut()->addSymbol(amd::Elf::RODATA, fmetadata.c_str(), - elfsymbol->SymInfo[NDX_METADATA].address, - elfsymbol->SymInfo[NDX_METADATA].size)) { - LogError("AddSymbol() failed to add fmetadata"); - return false; - } - } - continue; - } - amd::Elf::SymbolInfo* sinfo = &(elfsymbol->SymInfo[0]); - std::string FName = it.first; - - // For this kernel, get the demangled kernel name, which is used to identify each kernel. - const size_t name_sz = FName.size() - (sizeof(_kernel) - 1) - (sizeof(__OpenCL_) - 1); - std::string demangledKName = FName.substr(sizeof(__OpenCL_) - 1, name_sz); - - // Check if the current entry is valid - if (((sinfo[NDX_HEADER].size <= 0) || (sinfo[NDX_KERNEL].size <= 0)) && - (sinfo[NDX_AMDIL].size <= 0)) { - std::string tlog = - "Warning: both IL and CAL Image are not available for kernel " + demangledKName; - LogWarning(tlog.c_str()); - continue; - } - hasKernels = true; - - Kernel::InitData initData = {0}; - std::string ilSource(sinfo[NDX_AMDIL].address, sinfo[NDX_AMDIL].size); - std::string metadata(sinfo[NDX_METADATA].address, sinfo[NDX_METADATA].size); - if ((sinfo[NDX_HEADER].size <= 0) || (sinfo[NDX_KERNEL].size <= 0)) { - // IL recompilation - // TODO: global data recompilation as well. - // 1) parse IL; 2) parse metadata to set up kernel header - size_t pos; - if (!program.findAllILFuncs((programil.size() ? programil : ilSource), pos)) { - program.freeAllILFuncs(); - return false; - } - - bool isFailed = false; - for (uint32_t i = 0; i < program.funcs_.size(); ++i) { - ILFunc* func = program.funcs_[i]; - ElfSymbol_t* sym = tempObj.functionNameMap[func->name_]; - if (sym == NULL) { - // No metadata for this function. - continue; - } - - assert((func->metadata_.end_ == 0) && "ILFunc init failed"); - amd::Elf::SymbolInfo* si = &(sym->SymInfo[0]); - if (si[NDX_METADATA].size > 0) { - std::string meta(si[NDX_METADATA].address, si[NDX_METADATA].size); - if (!program.parseFuncMetadata(meta, 0, std::string::npos)) { - isFailed = true; - break; - } - if (func->metadata_.end_ != std::string::npos) { - assert(false && "ILFunc name and index does not match"); - isFailed = true; - break; - } - - // Accumulate all emulated local, region and private sizes, - // necessary for the kernel execution - initData.localSize_ += func->localSize_; - initData.privateSize_ += func->privateSize_; - - // Accumulate all HW local, region and private sizes, - // necessary for the kernel execution - initData.hwLocalSize_ += func->hwLocalSize_; - initData.hwPrivateSize_ += func->hwPrivateSize_; - initData.flags_ |= func->flags_; - } - } - - program.freeAllILFuncs(); - if (isFailed) { - return false; - } - } else { - KernelHeaderSymbol kHeader = {0}; - ::memcpy(&kHeader, sinfo[NDX_HEADER].address, (sizeof(kHeader) < sinfo[NDX_HEADER].size) - ? sizeof(kHeader) - : sinfo[NDX_HEADER].size); - - if (kHeader.version_ > VERSION_CURRENT) { - LogError("LoadKernelFromElf: cannot handle the newer version of the binary"); - return false; - } - - // VERSION_0 - initData.localSize_ = kHeader.localSize_; - initData.hwLocalSize_ = kHeader.hwLocalSize_; - initData.privateSize_ = kHeader.privateSize_; - initData.hwPrivateSize_ = kHeader.hwPrivateSize_; - initData.flags_ = kHeader.flags_; - } - - bool created; - NullKernel* gpuKernel = - program.createKernel(demangledKName, &initData, ilSource, metadata, &created, - sinfo[NDX_KERNEL].address, sinfo[NDX_KERNEL].size); - if (!created) { - std::string tlog = - "Error: Creating kernel during loading OCL binary " + demangledKName + " failed!"; - LogError(tlog.c_str()); - return false; - } - - recompiled = recompiled || (sinfo[NDX_KERNEL].size == 0); - - // Add the current kernel to the OCL binary in case recompilation happens and - // the new binary is needed. - if (!storeKernel(demangledKName, gpuKernel, &initData, metadata, ilSource)) { - return false; - } - } - - *hasRecompiled = recompiled; - return hasKernels; -} - -bool ClBinary::storeKernel(const std::string& name, const NullKernel* nullKernel, - Kernel::InitData* initData, const std::string& metadata, - const std::string& ilSource) { - if (!saveISA()) { - return true; - } - - // should we save kernel metadata only under saveAMDIL()? - bool kernelMetaStored = false; - - if (false && (ilSource.size() > 0)) { - // Save IL (this is the per-kernel IL) - std::string ilName = "__OpenCL_" + name + "_amdil"; - if (!elfOut()->addSymbol(amd::Elf::ILTEXT, ilName.c_str(), ilSource.data(), - ilSource.size())) { - LogError("AddElfSymbol failed"); - return false; - } - - std::string metaName = "__OpenCL_" + name + "_metadata"; - // Save metadata symbols in .rodata - if (!elfOut()->addSymbol(amd::Elf::RODATA, metaName.c_str(), metadata.data(), - metadata.size())) { - LogError("AddElfSymbol failed"); - return false; - } - kernelMetaStored = true; - } - - if (!saveISA()) { - return true; - } - - size_t binarySize = (nullKernel != NULL) ? nullKernel->getCalBinarySize() : 0; - if (binarySize != 0) { - if (!kernelMetaStored) { - std::string metaName = "__OpenCL_" + name + "_metadata"; - // Save metadata symbols in .rodata - if (!elfOut()->addSymbol(amd::Elf::RODATA, metaName.c_str(), metadata.data(), - metadata.size())) { - LogError("AddSymbol failed"); - return false; - } - } - // Save kernel symbol that is associated with GPU ISA - std::string kernelName = "__OpenCL_" + name + "_kernel"; - uint8_t* isacode = new uint8_t[binarySize]; - if (!nullKernel->getCalBinary(reinterpret_cast(isacode), binarySize)) { - LogError("Failed to read GPU kernel isa"); - delete[] isacode; - return false; - } - if (!elfOut()->addSymbol(amd::Elf::CAL, kernelName.c_str(), isacode, binarySize)) { - LogError("AddElfSymbol failed"); - return false; - } - delete[] isacode; - - // Save kernel header information into a pseudo symbol - // __OpenCL__header - // for example, given a kernel foo, this pseudo symbol - // would be __OpenCL_foo_header - std::string headerName = "__OpenCL_" + name + "_header"; - KernelHeaderSymbol kHeader; - // VERSION_0 - kHeader.privateSize_ = initData->privateSize_; - kHeader.localSize_ = initData->localSize_; - kHeader.regionSize_ = 0; - kHeader.hwPrivateSize_ = initData->hwPrivateSize_; - kHeader.hwLocalSize_ = initData->hwLocalSize_; - kHeader.hwRegionSize_ = 0; - kHeader.flags_ = initData->flags_; - - // VERSION_1 - kHeader.version_ = VERSION_CURRENT; - - if (!elfOut()->addSymbol(amd::Elf::RODATA, headerName.c_str(), &kHeader, sizeof(kHeader))) { - LogError("AddElfSymbol failed"); - return false; - } - } - return true; -} - -bool ClBinary::loadGlobalData(Program& program) { - const char __OpenCL_[] = "__OpenCL_"; - const char _global[] = "_global"; - - int num = elfIn()->getSymbolNum(); - for (int ndx = 0; ndx < num; ndx++) { - amd::Elf::SymbolInfo symInfo; - if (!elfIn()->getSymbolInfo(ndx, &symInfo)) { - LogError("LoadGlobalDataFromElf: getSymbolInfo() fails"); - return false; - } - - std::string globalName(symInfo.sym_name); - const size_t offset = sizeof(__OpenCL_) - 1; - if (globalName.compare(0, offset, __OpenCL_) != 0) { - continue; - } - const size_t suffixPos = globalName.rfind('_'); - if (globalName.compare(suffixPos, sizeof(_global) - 1, _global) != 0) { - continue; - } - - // Get index for this global - std::string indexString = globalName.substr(offset, suffixPos - offset); - uint index = ::atoi(indexString.c_str()); - - if (!program.allocGlobalData(symInfo.address, symInfo.size, index)) { - LogError("Couldn't load global data"); - return false; - } - } - - return true; -} - -bool ClBinary::storeGlobalData(const void* globalData, size_t dataSize, uint index) { - // For each global, use "__OpenCL_" as its name - // Since there is no name in amdil, just use "__OpenCL__global" for now. - std::stringstream glbName; - glbName << "__OpenCL_" << index << "_global"; - - if (!elfOut()->addSymbol(amd::Elf::RODATA, glbName.str().c_str(), globalData, dataSize)) { - LogError("addSymbol() failed"); - return false; - } - return true; -} - -bool ClBinary::clearElfOut() { - // Recreate libelf elf object - if (!elfOut()->Clear()) { - return false; - } - - // Need to re-setup target - return setElfTarget(); -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpubinary.hpp b/projects/clr/rocclr/device/gpu/gpubinary.hpp deleted file mode 100644 index 6a84035bdc..0000000000 --- a/projects/clr/rocclr/device/gpu/gpubinary.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUBINARY_HPP_ -#define GPUBINARY_HPP_ - -#include "top.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpukernel.hpp" - -namespace gpu { - -class ClBinary : public device::ClBinary { - public: -#pragma pack(push, 8) - // Kernel version in the ELF header symbol - enum KernelVersions { VERSION_0 = 0, VERSION_1, VERSION_CURRENT = VERSION_1 }; - - /* This is the ELF header symbol */ - struct KernelHeaderSymbol { - /* VERSION_0 - Version 0 has 8 uint32_t (32 bytes), top 5 are used, the rest zero'ed. - In Version_0, KernelHeaderSymbol is the same as KernelHeader - */ - uint32_t privateSize_; //!< Emulated private memory size - uint32_t localSize_; //!< Emulated local memory size - uint32_t hwPrivateSize_; //!< HW private memory size - uint32_t hwLocalSize_; //!< HW local memory size - uint32_t flags_; //!< Kernel's flags - - /* VERSION_1 - VERSION_1 has 6 uint32_t. - */ - uint32_t version_; //!< Kernel's version - uint32_t regionSize_; //!< Region memory size - uint32_t hwRegionSize_; //!< HW region memory size - - /* New entries can be added here, do not change the previous entries */ - }; - -#pragma pack(pop) - - //! Constructor - ClBinary(const NullDevice& dev, BinaryImageFormat bifVer = BIF_VERSION2) - : device::ClBinary(dev, bifVer) {} - - //! Destructor - ~ClBinary() {} - - //! Creates and loads kernels from the OCL ELF binary file into the program - bool loadKernels(NullProgram& program, //!< Program object with the binary - bool* hasRecompiled //!< Recompile amdil to isa. - ); - - //! Stores compiled kernel into the OCL ELF binary file - bool storeKernel(const std::string& name, //!< Kernel's name - const NullKernel* nullKernel, //!< The kernel to add - Kernel::InitData* initData, //!< Kernel init data - const std::string& metadata, //!< Kernel's metadata - const std::string& ilSource //!< IL source text - ); - - //! Loads the program's global data - bool loadGlobalData(Program& program //!< The program object for the global data load - ); - - //! Stores the program's global data - bool storeGlobalData(const void* globalData, //!< The program global data - size_t dataSize, //!< The program global data size - uint index //!< The global data storage index - ); - - //! Set elf header information for GPU target - bool setElfTarget() { - uint32_t target = static_cast(dev().calTarget()); - assert(((0xFFFF8000 & target) == 0) && "ASIC target ID >= 2^15"); - uint16_t elf_target = (uint16_t)(0x7FFF & target); - return elfOut()->setTarget(elf_target, amd::Elf::CAL_PLATFORM); - } - - //! Clear elf out. - bool clearElfOut(); - - private: - //! Disable default copy constructor - ClBinary(const ClBinary&); - - //! Disable default operator= - ClBinary& operator=(const ClBinary&); - - //! Returns the GPU device for this object - const NullDevice& dev() const { return static_cast(dev_); } -}; - -} // namespace gpu - -#endif // GPUBINARY_HPP_ diff --git a/projects/clr/rocclr/device/gpu/gpublit.cpp b/projects/clr/rocclr/device/gpu/gpublit.cpp deleted file mode 100644 index 56d179a2ce..0000000000 --- a/projects/clr/rocclr/device/gpu/gpublit.cpp +++ /dev/null @@ -1,2458 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "platform/commandqueue.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpublit.hpp" -#include "device/gpu/gpumemory.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "utils/debug.hpp" -#include - -namespace gpu { - -DmaBlitManager::DmaBlitManager(VirtualGPU& gpu, Setup setup) - : HostBlitManager(gpu, setup), - MinSizeForPinnedTransfer(dev().settings().pinnedMinXferSize_), - completeOperation_(false), - context_(NULL) {} - -inline void DmaBlitManager::synchronize() const { - if (syncOperation_) { - gpu().waitAllEngines(); - gpu().releaseMemObjects(); - } -} - -inline Memory& DmaBlitManager::gpuMem(device::Memory& mem) const { - return static_cast(mem); -} - -bool DmaBlitManager::readMemoryStaged(Memory& srcMemory, void* dstHost, Memory** xferBuf, - size_t origin, size_t& offset, size_t& totalSize, - size_t xferSize) const { - amd::Coord3D dst(0, 0, 0); - size_t tmpSize; - uint idxWrite = 0; - uint idxRead = 0; - size_t chunkSize; - static const bool CopyRect = false; - // Flush DMA for ASYNC copy - static const bool FlushDMA = true; - - if (dev().xferRead().bufSize() < 128 * Ki) { - chunkSize = dev().xferRead().bufSize(); - } else { - chunkSize = std::min(amd::alignUp(xferSize / 4, 256), dev().xferRead().bufSize()); - chunkSize = std::max(chunkSize, 128 * Ki); - } - - // Find the partial transfer size - tmpSize = std::min(chunkSize, xferSize); - - amd::Coord3D srcLast(origin + offset, 0, 0); - amd::Coord3D copySizeLast(tmpSize, 0, 0); - - // Copy data into the temporary surface - if (!srcMemory.partialMemCopyTo(gpu(), srcLast, dst, copySizeLast, *xferBuf[idxWrite], CopyRect, - FlushDMA)) { - return false; - } - - totalSize -= tmpSize; - xferSize -= tmpSize; - offset += tmpSize; - - while (xferSize != 0) { - // Find the partial transfer size - tmpSize = std::min(chunkSize, xferSize); - - amd::Coord3D src(origin + offset, 0, 0); - amd::Coord3D copySize(tmpSize, 0, 0); - - idxWrite = (idxWrite + 1) % 2; - // Copy data into the temporary surface - if (!srcMemory.partialMemCopyTo(gpu(), src, dst, copySize, *xferBuf[idxWrite], CopyRect, - FlushDMA)) { - return false; - } - - // Read previous buffer - if (!xferBuf[idxRead]->hostRead(&gpu(), - reinterpret_cast(dstHost) + offset - copySizeLast[0], - dst, copySizeLast)) { - return false; - } - idxRead = (idxRead + 1) % 2; - copySizeLast = copySize; - - totalSize -= tmpSize; - xferSize -= tmpSize; - offset += tmpSize; - } - - // Last read - if (!xferBuf[idxRead]->hostRead( - &gpu(), reinterpret_cast(dstHost) + offset - copySizeLast[0], dst, copySizeLast)) { - return false; - } - - return true; -} - -bool DmaBlitManager::readBuffer(device::Memory& srcMemory, void* dstHost, - const amd::Coord3D& origin, const amd::Coord3D& size, - bool entire) const { - // Use host copy if memory has direct access - if (setup_.disableReadBuffer_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable())) { - return HostBlitManager::readBuffer(srcMemory, dstHost, origin, size, entire); - } else { - size_t srcSize = size[0]; - size_t offset = 0; - size_t pinSize = dev().settings().pinnedXferSize_; - pinSize = std::min(pinSize, srcSize); - - // Check if a pinned transfer can be executed - if (pinSize && (srcSize > MinSizeForPinnedTransfer)) { - // Allign offset to 4K boundary (Vista/Win7 limitation) - char* tmpHost = const_cast( - amd::alignDown(reinterpret_cast(dstHost), PinnedMemoryAlignment)); - - // Find the partial size for unaligned copy - size_t partial = reinterpret_cast(dstHost) - tmpHost; - - amd::Memory* pinned = NULL; - bool first = true; - size_t tmpSize; - size_t pinAllocSize; - - // Copy memory, using pinning - while (srcSize > 0) { - // If it's the first iterarion, then readjust the copy size - // to include alignment - if (first) { - pinAllocSize = amd::alignUp(pinSize + partial, PinnedMemoryAlignment); - tmpSize = std::min(pinAllocSize - partial, srcSize); - first = false; - } else { - tmpSize = std::min(pinSize, srcSize); - pinAllocSize = amd::alignUp(tmpSize, PinnedMemoryAlignment); - partial = 0; - } - amd::Coord3D dst(partial, 0, 0); - amd::Coord3D srcPin(origin[0] + offset, 0, 0); - amd::Coord3D copySizePin(tmpSize, 0, 0); - size_t partial2; - - // Allocate a GPU resource for pinning - pinned = pinHostMemory(tmpHost, pinAllocSize, partial2); - - if (pinned != NULL) { - // Get device memory for this virtual device - Memory* dstMemory = dev().getGpuMemory(pinned); - - if (!gpuMem(srcMemory).partialMemCopyTo(gpu(), srcPin, dst, copySizePin, *dstMemory)) { - LogWarning("DmaBlitManager::readBuffer failed a pinned copy!"); - gpu().addPinnedMem(pinned); - break; - } - gpu().addPinnedMem(pinned); - } else { - LogWarning("DmaBlitManager::readBuffer failed to pin a resource!"); - break; - } - srcSize -= tmpSize; - offset += tmpSize; - tmpHost = reinterpret_cast(tmpHost) + tmpSize + partial; - } - } - - if (0 != srcSize) { - Memory& xferBuf0 = dev().xferRead().acquire(); - Memory& xferBuf1 = dev().xferRead().acquire(); - Memory* xferBuf[2] = {&xferBuf0, &xferBuf1}; - - // Read memory using a staged resource - if (!readMemoryStaged(gpuMem(srcMemory), dstHost, xferBuf, origin[0], offset, srcSize, - srcSize)) { - LogError("DmaBlitManager::readBuffer failed!"); - return false; - } - - dev().xferRead().release(gpu(), xferBuf1); - dev().xferRead().release(gpu(), xferBuf0); - } - } - - return true; -} - -bool DmaBlitManager::readBufferRect(device::Memory& srcMemory, void* dstHost, - const amd::BufferRect& bufRect, const amd::BufferRect& hostRect, - const amd::Coord3D& size, bool entire) const { - // Use host copy if memory has direct access - if (setup_.disableReadBufferRect_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable())) { - return HostBlitManager::readBufferRect(srcMemory, dstHost, bufRect, hostRect, size, entire); - } else { - Memory& xferBuf = dev().xferRead().acquire(); - - amd::Coord3D dst(0, 0, 0); - size_t tmpSize = 0; - size_t bufOffset; - size_t hostOffset; - size_t srcSize; - - for (size_t z = 0; z < size[2]; ++z) { - for (size_t y = 0; y < size[1]; ++y) { - srcSize = size[0]; - bufOffset = bufRect.offset(0, y, z); - hostOffset = hostRect.offset(0, y, z); - - while (srcSize != 0) { - // Find the partial transfer size - tmpSize = std::min(dev().xferRead().bufSize(), srcSize); - - amd::Coord3D src(bufOffset, 0, 0); - amd::Coord3D copySize(tmpSize, 0, 0); - - // Copy data into the temporary surface - if (!gpuMem(srcMemory).partialMemCopyTo(gpu(), src, dst, copySize, xferBuf, true)) { - LogError("DmaBlitManager::readBufferRect failed!"); - return false; - } - - if (!xferBuf.hostRead(&gpu(), reinterpret_cast(dstHost) + hostOffset, dst, - copySize)) { - LogError("DmaBlitManager::readBufferRect failed!"); - return false; - } - - srcSize -= tmpSize; - bufOffset += tmpSize; - hostOffset += tmpSize; - } - } - } - dev().xferRead().release(gpu(), xferBuf); - } - - return true; -} - -bool DmaBlitManager::readImage(device::Memory& srcMemory, void* dstHost, const amd::Coord3D& origin, - const amd::Coord3D& size, size_t rowPitch, size_t slicePitch, - bool entire) const { - if (setup_.disableReadImage_) { - return HostBlitManager::readImage(srcMemory, dstHost, origin, size, rowPitch, slicePitch, - entire); - } else { - //! @todo Add HW accelerated path - return HostBlitManager::readImage(srcMemory, dstHost, origin, size, rowPitch, slicePitch, - entire); - } - - return true; -} - -bool DmaBlitManager::writeMemoryStaged(const void* srcHost, Memory& dstMemory, Memory& xferBuf, - size_t origin, size_t& offset, size_t& totalSize, - size_t xferSize) const { - amd::Coord3D src(0, 0, 0); - size_t chunkSize; - static const bool CopyRect = false; - // Flush DMA for ASYNC copy - // @todo Blocking write requires a flush to start earlier, - // but currently VDI doesn't provide that info - static const bool FlushDMA = false; - - if (dev().xferRead().bufSize() < 128 * Ki) { - chunkSize = dev().xferWrite().bufSize(); - } else { - chunkSize = std::min(amd::alignUp(xferSize / 4, 256), dev().xferWrite().bufSize()); - chunkSize = std::max(chunkSize, 128 * Ki); - } - - while (xferSize != 0) { - // Find the partial transfer size - size_t tmpSize = std::min(chunkSize, xferSize); - amd::Coord3D dst(origin + offset, 0, 0); - amd::Coord3D copySize(tmpSize, 0, 0); - - // Copy data into the temporary buffer, using CPU - if (!xferBuf.hostWrite(&gpu(), reinterpret_cast(srcHost) + offset, src, copySize, - Resource::Discard)) { - return false; - } - - // Copy data into the original destination memory - if (!xferBuf.partialMemCopyTo(gpu(), src, dst, copySize, dstMemory, CopyRect, FlushDMA)) { - return false; - } - - totalSize -= tmpSize; - offset += tmpSize; - xferSize -= tmpSize; - } - - return true; -} - -bool DmaBlitManager::writeBuffer(const void* srcHost, device::Memory& dstMemory, - const amd::Coord3D& origin, const amd::Coord3D& size, - bool entire) const { - // Use host copy if memory has direct access or it's persistent - if (setup_.disableWriteBuffer_ || - (gpuMem(dstMemory).isHostMemDirectAccess() && - (gpuMem(dstMemory).memoryType() != Resource::ExternalPhysical)) || - gpuMem(dstMemory).isPersistentDirectMap()) { - return HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire); - } else { - size_t dstSize = size[0]; - size_t tmpSize = 0; - size_t offset = 0; - size_t pinSize = dev().settings().pinnedXferSize_; - pinSize = std::min(pinSize, dstSize); - - // Check if a pinned transfer can be executed - if (pinSize && (dstSize > MinSizeForPinnedTransfer)) { - // Allign offset to 4K boundary (Vista/Win7 limitation) - char* tmpHost = const_cast( - amd::alignDown(reinterpret_cast(srcHost), PinnedMemoryAlignment)); - - // Find the partial size for unaligned copy - size_t partial = reinterpret_cast(srcHost) - tmpHost; - - amd::Memory* pinned = NULL; - bool first = true; - size_t tmpSize; - size_t pinAllocSize; - - // Copy memory, using pinning - while (dstSize > 0) { - // If it's the first iterarion, then readjust the copy size - // to include alignment - if (first) { - pinAllocSize = amd::alignUp(pinSize + partial, PinnedMemoryAlignment); - tmpSize = std::min(pinAllocSize - partial, dstSize); - first = false; - } else { - tmpSize = std::min(pinSize, dstSize); - pinAllocSize = amd::alignUp(tmpSize, PinnedMemoryAlignment); - partial = 0; - } - amd::Coord3D src(partial, 0, 0); - amd::Coord3D dstPin(origin[0] + offset, 0, 0); - amd::Coord3D copySizePin(tmpSize, 0, 0); - size_t partial2; - - // Allocate a GPU resource for pinning - pinned = pinHostMemory(tmpHost, pinAllocSize, partial2); - - if (pinned != NULL) { - // Get device memory for this virtual device - Memory* srcMemory = dev().getGpuMemory(pinned); - - if (!srcMemory->partialMemCopyTo(gpu(), src, dstPin, copySizePin, gpuMem(dstMemory))) { - LogWarning("DmaBlitManager::writeBuffer failed a pinned copy!"); - gpu().addPinnedMem(pinned); - break; - } - gpu().addPinnedMem(pinned); - } else { - LogWarning("DmaBlitManager::writeBuffer failed to pin a resource!"); - break; - } - dstSize -= tmpSize; - offset += tmpSize; - tmpHost = reinterpret_cast(tmpHost) + tmpSize + partial; - } - } - - - if (dstSize != 0) { - Memory& xferBuf = dev().xferWrite().acquire(); - - // Write memory using a staged resource - if (!writeMemoryStaged(srcHost, gpuMem(dstMemory), xferBuf, origin[0], offset, dstSize, - dstSize)) { - LogError("DmaBlitManager::writeBuffer failed!"); - return false; - } - - gpu().addXferWrite(xferBuf); - } - } - - return true; -} - -bool DmaBlitManager::writeBufferRect(const void* srcHost, device::Memory& dstMemory, - const amd::BufferRect& hostRect, - const amd::BufferRect& bufRect, const amd::Coord3D& size, - bool entire) const { - // Use host copy if memory has direct access or it's persistent - if (setup_.disableWriteBufferRect_ || - (dstMemory.isHostMemDirectAccess() && - (gpuMem(dstMemory).memoryType() != Resource::ExternalPhysical)) || - gpuMem(dstMemory).isPersistentDirectMap()) { - return HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire); - } else { - Memory& xferBuf = dev().xferWrite().acquire(); - - amd::Coord3D src(0, 0, 0); - size_t tmpSize = 0; - size_t bufOffset; - size_t hostOffset; - size_t dstSize; - - for (size_t z = 0; z < size[2]; ++z) { - for (size_t y = 0; y < size[1]; ++y) { - dstSize = size[0]; - bufOffset = bufRect.offset(0, y, z); - hostOffset = hostRect.offset(0, y, z); - - while (dstSize != 0) { - // Find the partial transfer size - tmpSize = std::min(dev().xferWrite().bufSize(), dstSize); - - amd::Coord3D dst(bufOffset, 0, 0); - amd::Coord3D copySize(tmpSize, 0, 0); - - // Copy data into the temporary buffer, using CPU - if (!xferBuf.hostWrite(&gpu(), reinterpret_cast(srcHost) + hostOffset, src, - copySize, Resource::Discard)) { - LogError("DmaBlitManager::writeBufferRect failed!"); - return false; - } - - // Copy data into the original destination memory - if (!xferBuf.partialMemCopyTo(gpu(), src, dst, copySize, gpuMem(dstMemory))) { - LogError("DmaBlitManager::writeBufferRect failed!"); - return false; - } - - dstSize -= tmpSize; - bufOffset += tmpSize; - hostOffset += tmpSize; - } - } - } - gpu().addXferWrite(xferBuf); - } - - return true; -} - -bool DmaBlitManager::writeImage(const void* srcHost, device::Memory& dstMemory, - const amd::Coord3D& origin, const amd::Coord3D& size, - size_t rowPitch, size_t slicePitch, bool entire) const { - if (setup_.disableWriteImage_) { - return HostBlitManager::writeImage(srcHost, dstMemory, origin, size, rowPitch, slicePitch, - entire); - } else { - //! @todo Add HW accelerated path - return HostBlitManager::writeImage(srcHost, dstMemory, origin, size, rowPitch, slicePitch, - entire); - } - - return true; -} - -bool DmaBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire) const { - if (setup_.disableCopyBuffer_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable() && - !dev().settings().apuSystem_ && gpuMem(dstMemory).isHostMemDirectAccess())) { - return HostBlitManager::copyBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size); - } else { - return gpuMem(srcMemory).partialMemCopyTo(gpu(), srcOrigin, dstOrigin, size, gpuMem(dstMemory)); - } - - return true; -} - -bool DmaBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::BufferRect& srcRect, const amd::BufferRect& dstRect, - const amd::Coord3D& size, bool entire) const { - if (setup_.disableCopyBufferRect_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable() && - gpuMem(dstMemory).isHostMemDirectAccess() && - (gpuMem(dstMemory).memoryType() != Resource::ExternalPhysical))) { - return HostBlitManager::copyBufferRect(srcMemory, dstMemory, srcRect, dstRect, size, entire); - } else { - size_t srcOffset; - size_t dstOffset; - - uint bytesPerElement = 16; - bool optimalElementSize = false; - bool subWindowRectCopy = true; - - srcOffset = srcRect.offset(0, 0, 0); - dstOffset = dstRect.offset(0, 0, 0); - - while (bytesPerElement >= 1) { - if (((srcOffset % 4) == 0) && ((dstOffset % 4) == 0) && ((size[0] % bytesPerElement) == 0) && - ((srcRect.rowPitch_ % bytesPerElement) == 0) && - ((srcRect.slicePitch_ % bytesPerElement) == 0) && - ((dstRect.rowPitch_ % bytesPerElement) == 0) && - ((dstRect.slicePitch_ % bytesPerElement) == 0)) { - optimalElementSize = true; - break; - } - bytesPerElement = bytesPerElement >> 1; - } - - // 19 bit limit in HW in SI and 16 bit limit in CI+(we adjust the ElementSize to 4bytes but the - // packet still has 14bits) - size_t pitchLimit = dev().settings().ciPlus_ ? (0x3FFF * bytesPerElement) | 0xF : 0x7FFFF; - size_t sizeLimit = dev().settings().ciPlus_ ? (0x3FFF * bytesPerElement) | 0xF : 0x3FFF; - - if (!optimalElementSize || (srcRect.rowPitch_ > pitchLimit) || - (dstRect.rowPitch_ > pitchLimit) || (size[0] > sizeLimit) || // See above - (size[1] > 0x3fff) || // 14 bits limit in HW - (size[2] > 0x7ff)) { // 11 bits limit in HW - // Restriction with rectLinearDRMDMA packet - subWindowRectCopy = false; - } - - if (subWindowRectCopy) { - // Copy data with subwindow copy packet - if (!gpuMem(srcMemory).partialMemCopyTo( - gpu(), amd::Coord3D(srcOffset, srcRect.rowPitch_, srcRect.slicePitch_), - amd::Coord3D(dstOffset, dstRect.rowPitch_, dstRect.slicePitch_), size, - gpuMem(dstMemory), true, false, bytesPerElement)) { - LogError("copyBufferRect failed!"); - return false; - } - } else { - for (size_t z = 0; z < size[2]; ++z) { - for (size_t y = 0; y < size[1]; ++y) { - srcOffset = srcRect.offset(0, y, z); - dstOffset = dstRect.offset(0, y, z); - - amd::Coord3D src(srcOffset, 0, 0); - amd::Coord3D dst(dstOffset, 0, 0); - amd::Coord3D copySize(size[0], 0, 0); - - // Copy data - if (!gpuMem(srcMemory).partialMemCopyTo(gpu(), src, dst, copySize, gpuMem(dstMemory))) { - LogError("copyBufferRect failed!"); - return false; - } - } - } - } - } - return true; -} - -bool DmaBlitManager::copyImageToBuffer(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire, size_t rowPitch, - size_t slicePitch) const { - bool result = false; - - if (setup_.disableCopyImageToBuffer_) { - result = HostBlitManager::copyImageToBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - } else { - // Use CAL path for a transfer - result = - gpuMem(srcMemory).partialMemCopyTo(gpu(), srcOrigin, dstOrigin, size, gpuMem(dstMemory)); - - // Check if a HostBlit transfer is required - if (completeOperation_ && !result) { - result = HostBlitManager::copyImageToBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - } - } - - return result; -} - -bool DmaBlitManager::copyBufferToImage(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire, size_t rowPitch, - size_t slicePitch) const { - bool result = false; - - if (setup_.disableCopyBufferToImage_) { - result = HostBlitManager::copyBufferToImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - } else { - // Use CAL path for a transfer - result = - gpuMem(srcMemory).partialMemCopyTo(gpu(), srcOrigin, dstOrigin, size, gpuMem(dstMemory)); - - // Check if a HostBlit transfer is required - if (completeOperation_ && !result) { - result = HostBlitManager::copyBufferToImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - } - } - - return result; -} - -bool DmaBlitManager::copyImage(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire) const { - bool result = false; - - if (setup_.disableCopyImage_) { - return HostBlitManager::copyImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, entire); - } else { - //! @todo Add HW accelerated path - return HostBlitManager::copyImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, entire); - } - - return result; -} - -KernelBlitManager::KernelBlitManager(VirtualGPU& gpu, Setup setup) - : DmaBlitManager(gpu, setup), - program_(NULL), - constantBuffer_(NULL), - xferBufferSize_(0), - lockXferOps_(NULL) { - for (uint i = 0; i < BlitTotal; ++i) { - kernels_[i] = NULL; - } - - for (uint i = 0; i < MaxXferBuffers; ++i) { - xferBuffers_[i] = NULL; - } - - completeOperation_ = false; -} - -KernelBlitManager::~KernelBlitManager() { - for (uint i = 0; i < BlitTotal; ++i) { - if (NULL != kernels_[i]) { - kernels_[i]->release(); - } - } - if (NULL != program_) { - program_->release(); - } - - if (NULL != context_) { - // Release a dummy context - context_->release(); - } - - if (NULL != constantBuffer_) { - constantBuffer_->release(); - } - - for (uint i = 0; i < MaxXferBuffers; ++i) { - if (NULL != xferBuffers_[i]) { - xferBuffers_[i]->release(); - } - } - - delete lockXferOps_; -} - -bool KernelBlitManager::create(amd::Device& device) { - if (!createProgram(static_cast(device))) { - return false; - } - return true; -} - -bool KernelBlitManager::createProgram(Device& device) { - std::vector devices; - devices.push_back(&device); - - // Save context and program for this device - context_ = device.blitProgram()->context_; - context_->retain(); - program_ = device.blitProgram()->program_; - program_->retain(); - - bool result = false; - do { - // Create kernel objects for all blits - for (uint i = 0; i < BlitTotal; ++i) { - const amd::Symbol* symbol = program_->findSymbol(BlitName[i]); - if (symbol == NULL) { - break; - } - kernels_[i] = new amd::Kernel(*program_, *symbol, BlitName[i]); - if (kernels_[i] == NULL) { - break; - } - // Validate blit kernels for the scratch memory usage (pre SI) - if (!device.validateKernel(*kernels_[i], &gpu())) { - break; - } - } - - result = true; - } while (!result); - - // Create an internal constant buffer - constantBuffer_ = new (*context_) amd::Buffer(*context_, CL_MEM_ALLOC_HOST_PTR, 4 * Ki); - // Assign the constant buffer to the current virtual GPU - constantBuffer_->setVirtualDevice(&gpu()); - if ((constantBuffer_ != NULL) && !constantBuffer_->create(NULL)) { - constantBuffer_->release(); - constantBuffer_ = NULL; - return false; - } else if (constantBuffer_ == NULL) { - return false; - } - - if (dev().settings().xferBufSize_ > 0) { - xferBufferSize_ = dev().settings().xferBufSize_; - for (uint i = 0; i < MaxXferBuffers; ++i) { - // Create internal xfer buffers for image copy optimization - xferBuffers_[i] = new (*context_) amd::Buffer(*context_, 0, xferBufferSize_); - // Assign the xfer buffer to the current virtual GPU - xferBuffers_[i]->setVirtualDevice(&gpu()); - if ((xferBuffers_[i] != NULL) && !xferBuffers_[i]->create(NULL)) { - xferBuffers_[i]->release(); - xferBuffers_[i] = NULL; - return false; - } else if (xferBuffers_[i] == NULL) { - return false; - } - - //! @note Workaround for conformance allocation test. - //! Force GPU mem alloc. - //! Unaligned images require xfer optimization, - //! but deferred memory allocation can cause - //! virtual heap fragmentation for big allocations and - //! then fail the following test with 32 bit ISA, because - //! runtime runs out of 4GB space. - dev().getGpuMemory(xferBuffers_[i]); - } - } - - lockXferOps_ = new amd::Monitor("Transfer Ops Lock", true); - if (NULL == lockXferOps_) { - return false; - } - - return result; -} - -// The following data structures will be used for the view creations. -// Some formats has to be converted before a kernel blit operation -struct FormatConvertion { - uint32_t clOldType_; - uint32_t clNewType_; -}; - -// The list of rejected data formats and corresponding conversion -static constexpr FormatConvertion RejectedData[] = { - {CL_UNORM_INT8, CL_UNSIGNED_INT8}, {CL_UNORM_INT16, CL_UNSIGNED_INT16}, - {CL_SNORM_INT8, CL_UNSIGNED_INT8}, {CL_SNORM_INT16, CL_UNSIGNED_INT16}, - {CL_HALF_FLOAT, CL_UNSIGNED_INT16}, {CL_FLOAT, CL_UNSIGNED_INT32}, - {CL_SIGNED_INT8, CL_UNSIGNED_INT8}, {CL_SIGNED_INT16, CL_UNSIGNED_INT16}, - {CL_UNORM_INT_101010, CL_UNSIGNED_INT8}, {CL_SIGNED_INT32, CL_UNSIGNED_INT32}}; - -// The list of rejected channel's order and corresponding conversion -static constexpr FormatConvertion RejectedOrder[] = { - {CL_A, CL_R}, {CL_RA, CL_RG}, {CL_LUMINANCE, CL_R}, {CL_INTENSITY, CL_R}, - {CL_RGB, CL_RGBA}, {CL_BGRA, CL_RGBA}, {CL_ARGB, CL_RGBA}, {CL_sRGB, CL_RGBA}, - {CL_sRGBx, CL_RGBA}, {CL_sRGBA, CL_RGBA}, {CL_sBGRA, CL_RGBA}}; - -const uint RejectedFormatDataTotal = sizeof(RejectedData) / sizeof(FormatConvertion); -const uint RejectedFormatChannelTotal = sizeof(RejectedOrder) / sizeof(FormatConvertion); - -bool KernelBlitManager::copyBufferToImage(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, - const amd::Coord3D& dstOrigin, const amd::Coord3D& size, - bool entire, size_t rowPitch, size_t slicePitch) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - static const bool CopyRect = false; - // Flush DMA for ASYNC copy - static const bool FlushDMA = true; - size_t imgRowPitch = size[0] * gpuMem(dstMemory).elementSize(); - size_t imgSlicePitch = imgRowPitch * size[1]; - - if (setup_.disableCopyBufferToImage_) { - result = DmaBlitManager::copyBufferToImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - synchronize(); - return result; - } - // Check if buffer is in system memory with direct access - else if (gpuMem(srcMemory).isHostMemDirectAccess() && - (((rowPitch == 0) && (slicePitch == 0)) || - ((rowPitch == imgRowPitch) && ((slicePitch == 0) || (slicePitch == imgSlicePitch))))) { - // First attempt to do this all with DMA, - // but there are restriciton with older hardware - if (dev().settings().imageDMA_) { - result = DmaBlitManager::copyBufferToImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - if (result) { - synchronize(); - return result; - } - } - - if (!setup_.disableCopyBufferToImageOpt_) { - // Find the overall copy size - size_t copySize = size[0] * size[1] * size[2] * gpuMem(dstMemory).elementSize(); - - // Check if double copy was requested - if (xferBufferSize_ != 0) { - amd::Coord3D src(srcOrigin); - amd::Coord3D xferSrc(0, 0, 0); - amd::Coord3D dst(dstOrigin); - amd::Coord3D xferRect(size); - // Find transfer size in pixels - size_t xferSizePix = xferBufferSize_ / gpuMem(dstMemory).elementSize(); - bool transfer = true; - - // Find transfer rectangle - if (xferRect[0] > xferSizePix) { - // The algorithm can't break a line. - // It requires multiple rectangles tracking - transfer = false; - } else { - xferRect.c[1] = xferSizePix / xferRect[0]; - } - // Check if we exceeded the original size boundary in Y - if (xferRect[1] > size[1]) { - xferRect.c[1] = size[1]; - xferRect.c[2] = xferSizePix / (xferRect[0] * xferRect[1]); - } else { - xferRect.c[2] = 1; - } - // Check if we exceeded the original size boundary in Z - if (xferRect[2] > size[2]) { - xferRect.c[2] = size[2]; - } - // Make sure size in Y dimension is divided by the rectangle size - if (size[2] > 1) { - while ((size[1] % xferRect[1]) != 0) { - xferRect.c[1]--; - } - } - - // Find one step copy size, based on the copy rectange - amd::Coord3D oneStepSize(xferRect[0] * xferRect[1] * xferRect[2] * - gpuMem(dstMemory).elementSize()); - - // Initialize transfer buffer array - Memory* xferBuf[MaxXferBuffers]; - for (uint i = 0; i < MaxXferBuffers; ++i) { - xferBuf[i] = dev().getGpuMemory(xferBuffers_[i]); - if (xferBuf[i] == NULL) { - transfer = false; - break; - } - } - - // Loop until we transfer all data - while (transfer && (copySize > 0)) { - size_t copySizeTmp = copySize; - amd::Coord3D srcTmp(src); - amd::Coord3D oneStepSizeTmp(oneStepSize); - // Step 1. Initiate DRM transfer with all staging buffers - for (uint i = 0; i < MaxXferBuffers; ++i) { - // Make sure we don't transfer more than copy size - if (copySizeTmp > 0) { - if (!gpuMem(srcMemory).partialMemCopyTo(gpu(), srcTmp, xferSrc, oneStepSizeTmp, - *xferBuf[i], CopyRect, FlushDMA)) { - transfer = false; - break; - } - - copySizeTmp -= oneStepSizeTmp[0]; - // Change buffer offset - srcTmp.c[0] += oneStepSizeTmp[0]; - - if (copySizeTmp < oneStepSizeTmp[0]) { - oneStepSizeTmp.c[0] = copySizeTmp; - } - } else { - break; - } - } - - // Step 2. Initiate compute transfer with all staging buffers - for (uint i = 0; i < MaxXferBuffers; ++i) { - if (copySize > 0) { - if (!copyBufferToImageKernel(*xferBuf[i], dstMemory, xferSrc, dst, xferRect, false)) { - transfer = false; - break; - } - gpu().flushDMA(MainEngine); - - copySize -= oneStepSize[0]; - // Change buffer offset - src.c[0] += oneStepSize[0]; - // Change image offset, ignore X offset - for (uint j = 1; j < 3; ++j) { - dst.c[j] += xferRect[j]; - if ((dst[j] - dstOrigin[j]) >= size[j]) { - dst.c[j] = dstOrigin[j]; - } else { - break; - } - } - // Recalculate rectangle size if the remain data is smaller - if (copySize < oneStepSize[0]) { - for (uint j = 0; j < 3; ++j) { - xferRect.c[j] = size[j] - (dst[j] - dstOrigin[j]); - } - oneStepSize.c[0] = copySize; - } - } else { - break; - } - } - } - - if (copySize == 0) { - result = true; - } else { - LogWarning("2 step transfer in copyBufferToImage failed"); - } - } - } - } - - if (!result) { - result = copyBufferToImageKernel(srcMemory, dstMemory, srcOrigin, dstOrigin, size, entire, - rowPitch, slicePitch); - } - - synchronize(); - - return result; -} - -void CalcRowSlicePitches(uint64_t* pitch, const int32_t* copySize, size_t rowPitch, - size_t slicePitch, const Memory& mem) { - size_t memFmtSize = memoryFormatSize(mem.cal()->format_).size_; - bool img1Darray = (mem.cal()->dimension_ == GSL_MOA_TEXTURE_1D_ARRAY) ? true : false; - - if (rowPitch == 0) { - pitch[0] = copySize[0]; - } else { - pitch[0] = rowPitch / memFmtSize; - } - if (slicePitch == 0) { - pitch[1] = pitch[0] * (img1Darray ? 1 : copySize[1]); - } else { - pitch[1] = slicePitch / memFmtSize; - } - assert((pitch[0] <= pitch[1]) && "rowPitch must be <= slicePitch"); - - if (img1Darray) { - // For 1D array rowRitch = slicePitch - pitch[0] = pitch[1]; - } -} - -static void setArgument(amd::Kernel* kernel, size_t index, size_t size, const void* value) { - const amd::KernelParameterDescriptor& desc = kernel->signature().at(index); - - void* param = kernel->parameters().values() + desc.offset_; - assert((desc.type_ == T_POINTER || value != NULL || - (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL)) && - "not a valid local mem arg"); - - uint32_t uint32_value = 0; - uint64_t uint64_value = 0; - - if (desc.type_ == T_POINTER && (desc.addressQualifier_ != CL_KERNEL_ARG_ADDRESS_LOCAL)) { - if ((value == NULL) || (static_cast(value) == NULL)) { - LP64_SWITCH(uint32_value, uint64_value) = 0; - reinterpret_cast(kernel->parameters().values() + - kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = nullptr; - } else { - // convert cl_mem to amd::Memory*, return false if invalid. - LP64_SWITCH(uint32_value, uint64_value) = static_cast(( - *static_cast(value))->virtualAddress()); - reinterpret_cast(kernel->parameters().values() + - kernel->parameters().memoryObjOffset())[desc.info_.arrayIndex_] = - *static_cast(value); - } - } else if (desc.type_ == T_SAMPLER) { - assert(false && "No sampler support in blit manager! Use internal samplers!"); - } else - switch (desc.size_) { - case 4: - if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { - uint32_value = size; - } else { - uint32_value = *static_cast(value); - } - break; - case 8: - if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { - uint64_value = size; - } else { - uint64_value = *static_cast(value); - } - break; - default: - break; - } - switch (desc.size_) { - case sizeof(uint32_t): - *static_cast(param) = uint32_value; - break; - case sizeof(uint64_t): - *static_cast(param) = uint64_value; - break; - default: - ::memcpy(param, value, size); - break; - } -} - -bool KernelBlitManager::copyBufferToImageKernel(device::Memory& srcMemory, - device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, - const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire, - size_t rowPitch, size_t slicePitch) const { - bool rejected = false; - Memory* dstView = &gpuMem(dstMemory); - bool releaseView = false; - bool result = false; - CalFormat imgFormat; - imgFormat.channelOrder_ = gpuMem(dstMemory).cal()->channelOrder_; - imgFormat.type_ = gpuMem(dstMemory).cal()->format_; - amd::Image::Format newFormat(dev().getOclFormat(imgFormat)); - - // Find unsupported formats - for (uint i = 0; i < RejectedFormatDataTotal; ++i) { - if (RejectedData[i].clOldType_ == newFormat.image_channel_data_type) { - newFormat.image_channel_data_type = RejectedData[i].clNewType_; - rejected = true; - break; - } - } - - // Find unsupported channel's order - for (uint i = 0; i < RejectedFormatChannelTotal; ++i) { - if (RejectedOrder[i].clOldType_ == newFormat.image_channel_order) { - newFormat.image_channel_order = RejectedOrder[i].clNewType_; - rejected = true; - break; - } - } - - // If the image format was rejected, then attempt to create a view - if (rejected) { - dstView = createView(gpuMem(dstMemory), dev().getCalFormat(newFormat)); - if (dstView != NULL) { - rejected = false; - releaseView = true; - } - } - - // Fall into the host path if the image format was rejected - if (rejected) { - return HostBlitManager::copyBufferToImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire); - } - - // Use a common blit type with three dimensions by default - uint blitType = BlitCopyBufferToImage; - size_t dim = 0; - size_t globalWorkOffset[3] = {0, 0, 0}; - size_t globalWorkSize[3]; - size_t localWorkSize[3]; - - // Program the kernels workload depending on the blit dimensions - dim = 3; - if (gpuMem(dstMemory).cal()->dimSize_ == 1) { - globalWorkSize[0] = amd::alignUp(size[0], 256); - globalWorkSize[1] = amd::alignUp(size[1], 1); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = 256; - localWorkSize[1] = localWorkSize[2] = 1; - } else if (gpuMem(dstMemory).cal()->dimSize_ == 2) { - globalWorkSize[0] = amd::alignUp(size[0], 16); - globalWorkSize[1] = amd::alignUp(size[1], 16); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = localWorkSize[1] = 16; - localWorkSize[2] = 1; - } else { - globalWorkSize[0] = amd::alignUp(size[0], 8); - globalWorkSize[1] = amd::alignUp(size[1], 8); - globalWorkSize[2] = amd::alignUp(size[2], 4); - localWorkSize[0] = localWorkSize[1] = 8; - localWorkSize[2] = 4; - } - - // Program kernels arguments for the blit operation - Memory* mem = &gpuMem(srcMemory); - setArgument(kernels_[blitType], 0, sizeof(cl_mem), &mem); - mem = dstView; - setArgument(kernels_[blitType], 1, sizeof(cl_mem), &mem); - const MemFormatStruct& memFmt = memoryFormatSize(gpuMem(dstMemory).cal()->format_); - - // 1 element granularity for writes by default - int32_t granularity = 1; - if (memFmt.size_ == 2) { - granularity = 2; - } else if (memFmt.size_ >= 4) { - granularity = 4; - } - CondLog(((srcOrigin[0] % granularity) != 0), "Unaligned offset in blit!"); - uint64_t srcOrg[4] = {srcOrigin[0] / granularity, srcOrigin[1], srcOrigin[2], 0}; - setArgument(kernels_[blitType], 2, sizeof(srcOrg), srcOrg); - - int32_t dstOrg[4] = {(int32_t)dstOrigin[0], (int32_t)dstOrigin[1], (int32_t)dstOrigin[2], 0}; - int32_t copySize[4] = {(int32_t)size[0], (int32_t)size[1], (int32_t)size[2], 0}; - - setArgument(kernels_[blitType], 3, sizeof(dstOrg), dstOrg); - setArgument(kernels_[blitType], 4, sizeof(copySize), copySize); - - // Program memory format - uint multiplier = memFmt.size_ / sizeof(uint32_t); - multiplier = (multiplier == 0) ? 1 : multiplier; - uint32_t format[4] = {memFmt.components_, memFmt.size_ / memFmt.components_, multiplier, 0}; - setArgument(kernels_[blitType], 5, sizeof(format), format); - - // Program row and slice pitches - uint64_t pitch[4] = {0}; - CalcRowSlicePitches(pitch, copySize, rowPitch, slicePitch, gpuMem(dstMemory)); - setArgument(kernels_[blitType], 6, sizeof(pitch), pitch); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(dim, globalWorkOffset, globalWorkSize, localWorkSize); - - // Execute the blit - address parameters = kernels_[blitType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters); - if (releaseView) { - delete dstView; - } - - return result; -} - -bool KernelBlitManager::copyImageToBuffer(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, - const amd::Coord3D& dstOrigin, const amd::Coord3D& size, - bool entire, size_t rowPitch, size_t slicePitch) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - static const bool CopyRect = false; - // Flush DMA for ASYNC copy - static const bool FlushDMA = true; - size_t imgRowPitch = size[0] * gpuMem(srcMemory).elementSize(); - size_t imgSlicePitch = imgRowPitch * size[1]; - - if (setup_.disableCopyImageToBuffer_) { - result = HostBlitManager::copyImageToBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - synchronize(); - return result; - } - // Check if buffer is in system memory with direct access - else if (gpuMem(dstMemory).isHostMemDirectAccess() && - (((rowPitch == 0) && (slicePitch == 0)) || - ((rowPitch == imgRowPitch) && ((slicePitch == 0) || (slicePitch == imgSlicePitch))))) { - // First attempt to do this all with DMA, - // but there are restriciton with older hardware - // If the dest buffer is external physical(SDI), copy two step as - // single step SDMA is causing corruption and the cause is under investigation - if (dev().settings().imageDMA_ && - gpuMem(dstMemory).memoryType() != Resource::ExternalPhysical) { - result = DmaBlitManager::copyImageToBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire, rowPitch, slicePitch); - if (result) { - synchronize(); - return result; - } - } - - // Find the overall copy size - size_t copySize = size[0] * size[1] * size[2] * gpuMem(srcMemory).elementSize(); - - // Check if double copy was requested - if (xferBufferSize_ != 0) { - amd::Coord3D src(srcOrigin); - amd::Coord3D dst(dstOrigin); - amd::Coord3D xferDst(0, 0, 0); - amd::Coord3D xferRect(size); - // Find transfer size in pixels - size_t xferSizePix = xferBufferSize_ / gpuMem(srcMemory).elementSize(); - bool transfer = true; - - // Find transfer rectangle - if (xferRect[0] > xferSizePix) { - // The algorithm can't break a line. - // It requires multiple rectangles tracking - transfer = false; - } else { - xferRect.c[1] = xferSizePix / xferRect[0]; - } - // Check if we exceeded the original size boundary in Y - if (xferRect[1] > size[1]) { - xferRect.c[1] = size[1]; - xferRect.c[2] = xferSizePix / (xferRect[0] * xferRect[1]); - } else { - xferRect.c[2] = 1; - } - // Check if we exceeded the original size boundary in Z - if (xferRect[2] > size[2]) { - xferRect.c[2] = size[2]; - } - // Make sure size in Y dimension is divided by the rectangle size - if (size[2] > 1) { - while ((size[1] % xferRect[1]) != 0) { - xferRect.c[1]--; - } - } - - // Find one step copy size, based on the copy rectange - amd::Coord3D oneStepSize(xferRect[0] * xferRect[1] * xferRect[2] * - gpuMem(srcMemory).elementSize()); - - // Initialize transfer buffer array - Memory* xferBuf[MaxXferBuffers]; - for (uint i = 0; i < MaxXferBuffers; ++i) { - xferBuf[i] = dev().getGpuMemory(xferBuffers_[i]); - if (xferBuf[i] == NULL) { - transfer = false; - break; - } - } - - // Loop until we transfer all data - while (transfer && (copySize > 0)) { - size_t copySizeTmp = copySize; - amd::Coord3D srcTmp(src); - amd::Coord3D oneStepSizeTmp(oneStepSize); - amd::Coord3D xferRectTmp(xferRect); - - // Step 1. Initiate compute transfer with all staging buffers - for (uint i = 0; i < MaxXferBuffers; ++i) { - if (copySizeTmp > 0) { - if (!copyImageToBufferKernel(srcMemory, *xferBuf[i], srcTmp, xferDst, xferRectTmp, - false)) { - transfer = false; - break; - } - gpu().flushDMA(MainEngine); - - copySizeTmp -= oneStepSizeTmp[0]; - // Change image offset, ignore X offset - for (uint j = 1; j < 3; ++j) { - srcTmp.c[j] += xferRectTmp[j]; - if ((srcTmp[j] - srcOrigin[j]) >= size[j]) { - srcTmp.c[j] = srcOrigin[j]; - } else { - break; - } - } - // Recalculate rectangle size if the remain data is smaller - if (copySizeTmp < oneStepSizeTmp[0]) { - for (uint j = 0; j < 3; ++j) { - xferRectTmp.c[j] = size[j] - (srcTmp[j] - srcOrigin[j]); - } - } - } else { - break; - } - } - - // Step 2. Initiate DRM transfer with all staging buffers - for (uint i = 0; i < MaxXferBuffers; ++i) { - // Make sure we don't transfer more than copy size - if (copySize > 0) { - if (!xferBuf[i]->partialMemCopyTo(gpu(), xferDst, dst, oneStepSize, gpuMem(dstMemory), - CopyRect, FlushDMA)) { - transfer = false; - break; - } - - copySize -= oneStepSize[0]; - // Change buffer offset - dst.c[0] += oneStepSize[0]; - // Change image offset, ignore X offset - for (uint j = 1; j < 3; ++j) { - src.c[j] += xferRect[j]; - if ((src[j] - srcOrigin[j]) >= size[j]) { - src.c[j] = srcOrigin[j]; - } else { - break; - } - } - // Recalculate rectangle size if the remain data is smaller - if (copySize < oneStepSize[0]) { - for (uint j = 0; j < 3; ++j) { - xferRect.c[j] = size[j] - (src[j] - srcOrigin[j]); - } - oneStepSize.c[0] = copySize; - } - } else { - break; - } - } - } - - if (copySize == 0) { - result = true; - } else { - LogWarning("2 step transfer in copyBufferToImage failed"); - } - } - } - - if (!result) { - result = copyImageToBufferKernel(srcMemory, dstMemory, srcOrigin, dstOrigin, size, entire, - rowPitch, slicePitch); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::copyImageToBufferKernel(device::Memory& srcMemory, - device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, - const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire, - size_t rowPitch, size_t slicePitch) const { - bool rejected = false; - Memory* srcView = &gpuMem(srcMemory); - bool releaseView = false; - bool result = false; - CalFormat imgFormat; - imgFormat.channelOrder_ = gpuMem(srcMemory).cal()->channelOrder_; - imgFormat.type_ = gpuMem(srcMemory).cal()->format_; - amd::Image::Format newFormat(dev().getOclFormat(imgFormat)); - - // Find unsupported formats - for (uint i = 0; i < RejectedFormatDataTotal; ++i) { - if (RejectedData[i].clOldType_ == newFormat.image_channel_data_type) { - newFormat.image_channel_data_type = RejectedData[i].clNewType_; - rejected = true; - break; - } - } - - // Find unsupported channel's order - for (uint i = 0; i < RejectedFormatChannelTotal; ++i) { - if (RejectedOrder[i].clOldType_ == newFormat.image_channel_order) { - newFormat.image_channel_order = RejectedOrder[i].clNewType_; - rejected = true; - break; - } - } - - // If the image format was rejected, then attempt to create a view - if (rejected) { - srcView = createView(gpuMem(srcMemory), dev().getCalFormat(newFormat)); - if (srcView != NULL) { - rejected = false; - releaseView = true; - } - } - - // Fall into the host path if the image format was rejected - if (rejected) { - return HostBlitManager::copyImageToBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, size, - entire); - } - - uint blitType = BlitCopyImageToBuffer; - size_t dim = 0; - size_t globalWorkOffset[3] = {0, 0, 0}; - size_t globalWorkSize[3]; - size_t localWorkSize[3]; - - // Program the kernels workload depending on the blit dimensions - dim = 3; - // Find the current blit type - if (gpuMem(srcMemory).cal()->dimSize_ == 1) { - globalWorkSize[0] = amd::alignUp(size[0], 256); - globalWorkSize[1] = amd::alignUp(size[1], 1); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = 256; - localWorkSize[1] = localWorkSize[2] = 1; - } else if (gpuMem(srcMemory).cal()->dimSize_ == 2) { - globalWorkSize[0] = amd::alignUp(size[0], 16); - globalWorkSize[1] = amd::alignUp(size[1], 16); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = localWorkSize[1] = 16; - localWorkSize[2] = 1; - } else { - globalWorkSize[0] = amd::alignUp(size[0], 8); - globalWorkSize[1] = amd::alignUp(size[1], 8); - globalWorkSize[2] = amd::alignUp(size[2], 4); - localWorkSize[0] = localWorkSize[1] = 8; - localWorkSize[2] = 4; - } - - // Program kernels arguments for the blit operation - Memory* mem = srcView; - setArgument(kernels_[blitType], 0, sizeof(cl_mem), &mem); - mem = &gpuMem(dstMemory); - setArgument(kernels_[blitType], 1, sizeof(cl_mem), &mem); - - // Update extra paramters for USHORT and UBYTE pointers. - // Only then compiler can optimize the kernel to use - // UAV Raw for other writes - setArgument(kernels_[blitType], 2, sizeof(cl_mem), &mem); - setArgument(kernels_[blitType], 3, sizeof(cl_mem), &mem); - - int32_t srcOrg[4] = {(int32_t)srcOrigin[0], (int32_t)srcOrigin[1], (int32_t)srcOrigin[2], 0}; - int32_t copySize[4] = {(int32_t)size[0], (int32_t)size[1], (int32_t)size[2], 0}; - setArgument(kernels_[blitType], 4, sizeof(srcOrg), srcOrg); - const MemFormatStruct& memFmt = memoryFormatSize(gpuMem(srcMemory).cal()->format_); - - // 1 element granularity for writes by default - int32_t granularity = 1; - if (memFmt.size_ == 2) { - granularity = 2; - } else if (memFmt.size_ >= 4) { - granularity = 4; - } - CondLog(((dstOrigin[0] % granularity) != 0), "Unaligned offset in blit!"); - uint64_t dstOrg[4] = {dstOrigin[0] / granularity, dstOrigin[1], dstOrigin[2], 0}; - setArgument(kernels_[blitType], 5, sizeof(dstOrg), dstOrg); - setArgument(kernels_[blitType], 6, sizeof(copySize), copySize); - - // Program memory format - uint multiplier = memFmt.size_ / sizeof(uint32_t); - multiplier = (multiplier == 0) ? 1 : multiplier; - uint32_t format[4] = {memFmt.components_, memFmt.size_ / memFmt.components_, multiplier, 0}; - setArgument(kernels_[blitType], 7, sizeof(format), format); - - // Program row and slice pitches - uint64_t pitch[4] = {0}; - CalcRowSlicePitches(pitch, copySize, rowPitch, slicePitch, gpuMem(srcMemory)); - setArgument(kernels_[blitType], 8, sizeof(pitch), pitch); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(dim, globalWorkOffset, globalWorkSize, localWorkSize); - - // Execute the blit - address parameters = kernels_[blitType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters); - if (releaseView) { - delete srcView; - } - - return result; -} - -bool KernelBlitManager::copyImage(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin, - const amd::Coord3D& size, bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool rejected = false; - Memory* srcView = &gpuMem(srcMemory); - Memory* dstView = &gpuMem(dstMemory); - bool releaseView = false; - bool result = false; - CalFormat imgFormat; - imgFormat.channelOrder_ = gpuMem(srcMemory).cal()->channelOrder_; - imgFormat.type_ = gpuMem(srcMemory).cal()->format_; - amd::Image::Format newFormat(dev().getOclFormat(imgFormat)); - - // Find unsupported formats - for (uint i = 0; i < RejectedFormatDataTotal; ++i) { - if (RejectedData[i].clOldType_ == newFormat.image_channel_data_type) { - newFormat.image_channel_data_type = RejectedData[i].clNewType_; - rejected = true; - break; - } - } - - // Search for the rejected channel's order only if the format was rejected - // Note: Image blit is independent from the channel order - if (rejected) { - for (uint i = 0; i < RejectedFormatChannelTotal; ++i) { - if (RejectedOrder[i].clOldType_ == newFormat.image_channel_order) { - newFormat.image_channel_order = RejectedOrder[i].clNewType_; - rejected = true; - break; - } - } - } - - // Attempt to create a view if the format was rejected - if (rejected) { - srcView = createView(gpuMem(srcMemory), dev().getCalFormat(newFormat)); - if (srcView != NULL) { - dstView = createView(gpuMem(dstMemory), dev().getCalFormat(newFormat)); - if (dstView != NULL) { - rejected = false; - releaseView = true; - } else { - delete srcView; - } - } - } - - // Fall into the host path for the entire 2D copy or - // if the image format was rejected - if (rejected) { - result = HostBlitManager::copyImage(srcMemory, dstMemory, srcOrigin, dstOrigin, size, entire); - synchronize(); - return result; - } - - uint blitType = BlitCopyImage; - size_t dim = 0; - size_t globalWorkOffset[3] = {0, 0, 0}; - size_t globalWorkSize[3]; - size_t localWorkSize[3]; - - // Program the kernels workload depending on the blit dimensions - dim = 3; - // Find the current blit type - if ((gpuMem(srcMemory).cal()->dimSize_ == 1) || (gpuMem(dstMemory).cal()->dimSize_ == 1)) { - globalWorkSize[0] = amd::alignUp(size[0], 256); - globalWorkSize[1] = amd::alignUp(size[1], 1); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = 256; - localWorkSize[1] = localWorkSize[2] = 1; - } else if ((gpuMem(srcMemory).cal()->dimSize_ == 2) || (gpuMem(dstMemory).cal()->dimSize_ == 2)) { - globalWorkSize[0] = amd::alignUp(size[0], 16); - globalWorkSize[1] = amd::alignUp(size[1], 16); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = localWorkSize[1] = 16; - localWorkSize[2] = 1; - } else { - globalWorkSize[0] = amd::alignUp(size[0], 8); - globalWorkSize[1] = amd::alignUp(size[1], 8); - globalWorkSize[2] = amd::alignUp(size[2], 4); - localWorkSize[0] = localWorkSize[1] = 8; - localWorkSize[2] = 4; - } - - // The current OpenCL spec allows "copy images from a 1D image - // array object to a 1D image array object" only. - if ((gpuMem(srcMemory).cal()->dimension_ == GSL_MOA_TEXTURE_1D_ARRAY) || - (gpuMem(dstMemory).cal()->dimension_ == GSL_MOA_TEXTURE_1D_ARRAY)) { - blitType = BlitCopyImage1DA; - } - - // Program kernels arguments for the blit operation - Memory* mem = srcView; - setArgument(kernels_[blitType], 0, sizeof(cl_mem), &mem); - mem = dstView; - setArgument(kernels_[blitType], 1, sizeof(cl_mem), &mem); - - // Program source origin - int32_t srcOrg[4] = {(int32_t)srcOrigin[0], (int32_t)srcOrigin[1], (int32_t)srcOrigin[2], 0}; - setArgument(kernels_[blitType], 2, sizeof(srcOrg), srcOrg); - - // Program destinaiton origin - int32_t dstOrg[4] = {(int32_t)dstOrigin[0], (int32_t)dstOrigin[1], (int32_t)dstOrigin[2], 0}; - setArgument(kernels_[blitType], 3, sizeof(dstOrg), dstOrg); - - int32_t copySize[4] = {(int32_t)size[0], (int32_t)size[1], (int32_t)size[2], 0}; - setArgument(kernels_[blitType], 4, sizeof(copySize), copySize); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(dim, globalWorkOffset, globalWorkSize, localWorkSize); - - // Execute the blit - address parameters = kernels_[blitType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters); - if (releaseView) { - delete srcView; - delete dstView; - } - - synchronize(); - - return result; -} - -void FindPinSize(size_t& pinSize, const amd::Coord3D& size, size_t& rowPitch, size_t& slicePitch, - const Memory& mem) { - pinSize = size[0] * mem.elementSize(); - if ((rowPitch == 0) || (rowPitch == pinSize)) { - rowPitch = 0; - } else { - pinSize = rowPitch; - } - - // Calculate the pin size, which should be equal to the copy size - for (uint i = 1; i < mem.cal()->dimSize_; ++i) { - pinSize *= size[i]; - if (i == 1) { - if ((slicePitch == 0) || (slicePitch == pinSize)) { - slicePitch = 0; - } else { - if (mem.cal()->dimension_ != GSL_MOA_TEXTURE_1D_ARRAY) { - pinSize = slicePitch; - } else { - pinSize = slicePitch * size[i]; - } - } - } - } -} - -bool KernelBlitManager::readImage(device::Memory& srcMemory, void* dstHost, - const amd::Coord3D& origin, const amd::Coord3D& size, - size_t rowPitch, size_t slicePitch, bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host copy if memory has direct access or it's persistent - if (setup_.disableReadImage_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable())) { - result = - HostBlitManager::readImage(srcMemory, dstHost, origin, size, rowPitch, slicePitch, entire); - synchronize(); - return result; - } else { - size_t pinSize; - FindPinSize(pinSize, size, rowPitch, slicePitch, gpuMem(srcMemory)); - - size_t partial; - amd::Memory* amdMemory = pinHostMemory(dstHost, pinSize, partial); - - if (amdMemory == NULL) { - // Force SW copy - result = HostBlitManager::readImage(srcMemory, dstHost, origin, size, rowPitch, slicePitch, - entire); - synchronize(); - return result; - } - - // Readjust destination offset - const amd::Coord3D dstOrigin(partial); - - // Get device memory for this virtual device - Memory* dstMemory = dev().getGpuMemory(amdMemory); - - // Copy image to buffer - result = copyImageToBuffer(srcMemory, *dstMemory, origin, dstOrigin, size, entire, rowPitch, - slicePitch); - - // Add pinned memory for a later release - gpu().addPinnedMem(amdMemory); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::writeImage(const void* srcHost, device::Memory& dstMemory, - const amd::Coord3D& origin, const amd::Coord3D& size, - size_t rowPitch, size_t slicePitch, bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host copy if memory has direct access or it's persistent - if (setup_.disableWriteImage_ || gpuMem(dstMemory).isHostMemDirectAccess() || - gpuMem(dstMemory).isPersistentDirectMap()) { - result = - HostBlitManager::writeImage(srcHost, dstMemory, origin, size, rowPitch, slicePitch, entire); - synchronize(); - return result; - } else { - size_t pinSize; - FindPinSize(pinSize, size, rowPitch, slicePitch, gpuMem(dstMemory)); - - size_t partial; - amd::Memory* amdMemory = pinHostMemory(srcHost, pinSize, partial); - - if (amdMemory == NULL) { - // Force SW copy - result = HostBlitManager::writeImage(srcHost, dstMemory, origin, size, rowPitch, slicePitch, - entire); - synchronize(); - return result; - } - - // Readjust destination offset - const amd::Coord3D srcOrigin(partial); - - // Get device memory for this virtual device - Memory* srcMemory = dev().getGpuMemory(amdMemory); - - // Copy image to buffer - result = copyBufferToImage(*srcMemory, dstMemory, srcOrigin, origin, size, entire, rowPitch, - slicePitch); - - // Add pinned memory for a later release - gpu().addPinnedMem(amdMemory); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::copyBufferRect(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::BufferRect& srcRectIn, - const amd::BufferRect& dstRectIn, const amd::Coord3D& sizeIn, - bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - bool rejected = false; - - // Fall into the CAL path for rejected transfers - if (setup_.disableCopyBufferRect_ || gpuMem(srcMemory).isHostMemDirectAccess() || - gpuMem(dstMemory).isHostMemDirectAccess()) { - result = - DmaBlitManager::copyBufferRect(srcMemory, dstMemory, srcRectIn, dstRectIn, sizeIn, entire); - - if (result) { - synchronize(); - return result; - } - } - - uint blitType = BlitCopyBufferRect; - size_t dim = 3; - size_t globalWorkOffset[3] = {0, 0, 0}; - size_t globalWorkSize[3]; - size_t localWorkSize[3]; - - const static uint CopyRectAlignment[3] = {16, 4, 1}; - - uint i; - for (i = 0; i < sizeof(CopyRectAlignment) / sizeof(uint); i++) { - // Check source alignments - bool aligned = ((srcRectIn.rowPitch_ % CopyRectAlignment[i]) == 0); - aligned &= ((srcRectIn.slicePitch_ % CopyRectAlignment[i]) == 0); - aligned &= ((srcRectIn.start_ % CopyRectAlignment[i]) == 0); - - // Check destination alignments - aligned &= ((dstRectIn.rowPitch_ % CopyRectAlignment[i]) == 0); - aligned &= ((dstRectIn.slicePitch_ % CopyRectAlignment[i]) == 0); - aligned &= ((dstRectIn.start_ % CopyRectAlignment[i]) == 0); - - // Check copy size alignment in the first dimension - aligned &= ((sizeIn[0] % CopyRectAlignment[i]) == 0); - - if (aligned) { - if (CopyRectAlignment[i] != 1) { - blitType = BlitCopyBufferRectAligned; - } - break; - } - } - - amd::BufferRect srcRect; - amd::BufferRect dstRect; - amd::Coord3D size(sizeIn[0], sizeIn[1], sizeIn[2]); - - srcRect.rowPitch_ = srcRectIn.rowPitch_ / CopyRectAlignment[i]; - srcRect.slicePitch_ = srcRectIn.slicePitch_ / CopyRectAlignment[i]; - srcRect.start_ = srcRectIn.start_ / CopyRectAlignment[i]; - srcRect.end_ = srcRectIn.end_ / CopyRectAlignment[i]; - - dstRect.rowPitch_ = dstRectIn.rowPitch_ / CopyRectAlignment[i]; - dstRect.slicePitch_ = dstRectIn.slicePitch_ / CopyRectAlignment[i]; - dstRect.start_ = dstRectIn.start_ / CopyRectAlignment[i]; - dstRect.end_ = dstRectIn.end_ / CopyRectAlignment[i]; - - size.c[0] /= CopyRectAlignment[i]; - - // Program the kernel's workload depending on the transfer dimensions - if ((size[1] == 1) && (size[2] == 1)) { - globalWorkSize[0] = amd::alignUp(size[0], 256); - globalWorkSize[1] = 1; - globalWorkSize[2] = 1; - localWorkSize[0] = 256; - localWorkSize[1] = 1; - localWorkSize[2] = 1; - } else if (size[2] == 1) { - globalWorkSize[0] = amd::alignUp(size[0], 16); - globalWorkSize[1] = amd::alignUp(size[1], 16); - globalWorkSize[2] = 1; - localWorkSize[0] = localWorkSize[1] = 16; - localWorkSize[2] = 1; - } else { - globalWorkSize[0] = amd::alignUp(size[0], 8); - globalWorkSize[1] = amd::alignUp(size[1], 8); - globalWorkSize[2] = amd::alignUp(size[2], 4); - localWorkSize[0] = localWorkSize[1] = 8; - localWorkSize[2] = 4; - } - - - // Program kernels arguments for the blit operation - Memory* mem = &gpuMem(srcMemory); - setArgument(kernels_[blitType], 0, sizeof(cl_mem), &mem); - mem = &gpuMem(dstMemory); - setArgument(kernels_[blitType], 1, sizeof(cl_mem), &mem); - uint64_t src[4] = {srcRect.rowPitch_, srcRect.slicePitch_, srcRect.start_, 0}; - setArgument(kernels_[blitType], 2, sizeof(src), src); - uint64_t dst[4] = {dstRect.rowPitch_, dstRect.slicePitch_, dstRect.start_, 0}; - setArgument(kernels_[blitType], 3, sizeof(dst), dst); - uint64_t copySize[4] = {size[0], size[1], size[2], CopyRectAlignment[i]}; - setArgument(kernels_[blitType], 4, sizeof(copySize), copySize); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(dim, globalWorkOffset, globalWorkSize, localWorkSize); - - // Execute the blit - address parameters = kernels_[blitType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters); - - synchronize(); - - return result; -} - -bool KernelBlitManager::readBuffer(device::Memory& srcMemory, void* dstHost, - const amd::Coord3D& origin, const amd::Coord3D& size, - bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - // Use host copy if memory has direct access - if (setup_.disableReadBuffer_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable())) { - result = HostBlitManager::readBuffer(srcMemory, dstHost, origin, size, entire); - synchronize(); - return result; - } else { - size_t pinSize = size[0]; - // Check if a pinned transfer can be executed with a single pin - if ((pinSize <= dev().settings().pinnedXferSize_) && (pinSize > MinSizeForPinnedTransfer)) { - size_t partial; - amd::Memory* amdMemory = pinHostMemory(dstHost, pinSize, partial); - - if (amdMemory == NULL) { - // Force SW copy - result = HostBlitManager::readBuffer(srcMemory, dstHost, origin, size, entire); - synchronize(); - return result; - } - - // Readjust host mem offset - amd::Coord3D dstOrigin(partial); - - // Get device memory for this virtual device - Memory* dstMemory = dev().getGpuMemory(amdMemory); - - // Copy image to buffer - result = copyBuffer(srcMemory, *dstMemory, origin, dstOrigin, size, entire); - - // Add pinned memory for a later release - gpu().addPinnedMem(amdMemory); - } else { - result = DmaBlitManager::readBuffer(srcMemory, dstHost, origin, size, entire); - } - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::readBufferRect(device::Memory& srcMemory, void* dstHost, - const amd::BufferRect& bufRect, - const amd::BufferRect& hostRect, const amd::Coord3D& size, - bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host copy if memory has direct access - if (setup_.disableReadBufferRect_ || - (gpuMem(srcMemory).isHostMemDirectAccess() && gpuMem(srcMemory).isCacheable())) { - result = HostBlitManager::readBufferRect(srcMemory, dstHost, bufRect, hostRect, size, entire); - synchronize(); - return result; - } else { - size_t pinSize = hostRect.start_ + hostRect.end_; - size_t partial; - amd::Memory* amdMemory = pinHostMemory(dstHost, pinSize, partial); - - if (amdMemory == NULL) { - // Force SW copy - result = HostBlitManager::readBufferRect(srcMemory, dstHost, bufRect, hostRect, size, entire); - synchronize(); - return result; - } - - // Readjust host mem offset - amd::BufferRect rect; - rect.rowPitch_ = hostRect.rowPitch_; - rect.slicePitch_ = hostRect.slicePitch_; - rect.start_ = hostRect.start_ + partial; - rect.end_ = hostRect.end_; - - // Get device memory for this virtual device - Memory* dstMemory = dev().getGpuMemory(amdMemory); - - // Copy image to buffer - result = copyBufferRect(srcMemory, *dstMemory, bufRect, rect, size, entire); - - // Add pinned memory for a later release - gpu().addPinnedMem(amdMemory); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::writeBuffer(const void* srcHost, device::Memory& dstMemory, - const amd::Coord3D& origin, const amd::Coord3D& size, - bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host copy if memory has direct access or it's persistent - if (setup_.disableWriteBuffer_ || - (gpuMem(dstMemory).isHostMemDirectAccess() && - (gpuMem(dstMemory).memoryType() != Resource::ExternalPhysical)) || - (gpuMem(dstMemory).memoryType() == Resource::Persistent)) { - result = HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire); - synchronize(); - return result; - } else { - size_t pinSize = size[0]; - - // Check if a pinned transfer can be executed with a single pin - if ((pinSize <= dev().settings().pinnedXferSize_) && (pinSize > MinSizeForPinnedTransfer)) { - size_t partial; - amd::Memory* amdMemory = pinHostMemory(srcHost, pinSize, partial); - - if (amdMemory == NULL) { - // Force SW copy - result = HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire); - synchronize(); - return result; - } - - // Readjust destination offset - const amd::Coord3D srcOrigin(partial); - - // Get device memory for this virtual device - Memory* srcMemory = dev().getGpuMemory(amdMemory); - - // Copy buffer rect - result = copyBuffer(*srcMemory, dstMemory, srcOrigin, origin, size, entire); - - // Add pinned memory for a later release - gpu().addPinnedMem(amdMemory); - } else { - result = DmaBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire); - } - } - - synchronize(); - - - return result; -} - -bool KernelBlitManager::writeBufferRect(const void* srcHost, device::Memory& dstMemory, - const amd::BufferRect& hostRect, - const amd::BufferRect& bufRect, const amd::Coord3D& size, - bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host copy if memory has direct access or it's persistent - if (setup_.disableWriteBufferRect_ || - (gpuMem(dstMemory).isHostMemDirectAccess() && - (gpuMem(dstMemory).memoryType() != Resource::ExternalPhysical)) || - gpuMem(dstMemory).isPersistentDirectMap()) { - result = HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire); - synchronize(); - return result; - } else { - size_t pinSize = hostRect.start_ + hostRect.end_; - size_t partial; - amd::Memory* amdMemory = pinHostMemory(srcHost, pinSize, partial); - - if (amdMemory == NULL) { - // Force SW copy - result = - HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire); - synchronize(); - return result; - } - - // Readjust destination offset - const amd::Coord3D srcOrigin(partial); - - // Get device memory for this virtual device - Memory* srcMemory = dev().getGpuMemory(amdMemory); - - // Readjust host mem offset - amd::BufferRect rect; - rect.rowPitch_ = hostRect.rowPitch_; - rect.slicePitch_ = hostRect.slicePitch_; - rect.start_ = hostRect.start_ + partial; - rect.end_ = hostRect.end_; - - // Copy buffer rect - result = copyBufferRect(*srcMemory, dstMemory, rect, bufRect, size, entire); - - // Add pinned memory for a later release - gpu().addPinnedMem(amdMemory); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::fillBuffer(device::Memory& memory, const void* pattern, size_t patternSize, - const amd::Coord3D& origin, const amd::Coord3D& size, - bool entire, bool forceBlit) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host fill if memory has direct access - if (setup_.disableFillBuffer_ || (!forceBlit && gpuMem(memory).isHostMemDirectAccess())) { - result = HostBlitManager::fillBuffer(memory, pattern, patternSize, origin, size, entire); - synchronize(); - return result; - } else { - uint fillType = FillBufferAligned; - size_t globalWorkOffset[3] = {0, 0, 0}; - uint64_t fillSize = size[0] / patternSize; - size_t globalWorkSize = amd::alignUp(fillSize, 256); - size_t localWorkSize = 256; - uint32_t alignment = (patternSize & 0x7) == 0 ? - sizeof(uint64_t) : - (patternSize & 0x3) == 0 ? - sizeof(uint32_t) : - (patternSize & 0x1) == 0 ? - sizeof(uint16_t) : sizeof(uint8_t); - - // Program kernels arguments for the fill operation - Memory* mem = &gpuMem(memory); - if (alignment == sizeof(uint64_t)) { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), &mem); - } else if (alignment == sizeof(uint32_t)) { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); - } else if (alignment == sizeof(uint16_t)) { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); - } else { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); - } - Memory* gpuCB = dev().getGpuMemory(constantBuffer_); - if (gpuCB == NULL) { - return false; - } - void* constBuf = gpuCB->map(&gpu(), Resource::WriteOnly); - memcpy(constBuf, pattern, patternSize); - gpuCB->unmap(&gpu()); - setArgument(kernels_[fillType], 4, sizeof(cl_mem), &gpuCB); - uint64_t offset = origin[0]; - - patternSize/= alignment; - offset /= alignment; - - setArgument(kernels_[fillType], 5, sizeof(uint32_t), &patternSize); - setArgument(kernels_[fillType], 6, sizeof(offset), &offset); - setArgument(kernels_[fillType], 7, sizeof(fillSize), &fillSize); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(1, globalWorkOffset, &globalWorkSize, &localWorkSize); - - // Execute the blit - address parameters = kernels_[fillType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[fillType], parameters); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::copyBuffer(device::Memory& srcMemory, device::Memory& dstMemory, - const amd::Coord3D& srcOrigin, const amd::Coord3D& dstOrigin, - const amd::Coord3D& sizeIn, bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - if (!gpuMem(srcMemory).isHostMemDirectAccess() && !gpuMem(dstMemory).isHostMemDirectAccess()) { - uint blitType = BlitCopyBuffer; - size_t dim = 1; - size_t globalWorkOffset[3] = {0, 0, 0}; - size_t globalWorkSize = 0; - size_t localWorkSize = 0; - - const static uint CopyBuffAlignment[3] = {16, 4, 1}; - amd::Coord3D size(sizeIn[0], sizeIn[1], sizeIn[2]); - - bool aligned; - uint i; - for (i = 0; i < sizeof(CopyBuffAlignment) / sizeof(uint); i++) { - // Check source alignments - aligned = ((srcOrigin[0] % CopyBuffAlignment[i]) == 0); - // Check destination alignments - aligned &= ((dstOrigin[0] % CopyBuffAlignment[i]) == 0); - // Check copy size alignment in the first dimension - aligned &= ((sizeIn[0] % CopyBuffAlignment[i]) == 0); - - if (aligned) { - if (CopyBuffAlignment[i] != 1) { - blitType = BlitCopyBufferAligned; - } - break; - } - } - - uint32_t remain; - if (blitType == BlitCopyBufferAligned) { - size.c[0] /= CopyBuffAlignment[i]; - } else { - if (dev().settings().ciPlus_) { - remain = size[0] % 4; - size.c[0] /= 4; - size.c[0] += 1; - } else { - // Check if offsets are aligned - aligned = ((srcOrigin[0] % sizeof(uint32_t)) == 0); - aligned &= ((dstOrigin[0] % sizeof(uint32_t)) == 0); - if (aligned) { - remain = size[0] % 4; - size.c[0] /= 4; - size.c[0] += 1; - } else { - remain = 8; - } - } - } - - // Program the dispatch dimensions - localWorkSize = 256; - globalWorkSize = amd::alignUp(size[0], 256); - - // Program kernels arguments for the blit operation - Memory* mem = &gpuMem(srcMemory); - setArgument(kernels_[blitType], 0, sizeof(cl_mem), &mem); - mem = &gpuMem(dstMemory); - setArgument(kernels_[blitType], 1, sizeof(cl_mem), &mem); - // Program source origin - uint64_t srcOffset = srcOrigin[0] / CopyBuffAlignment[i]; - ; - setArgument(kernels_[blitType], 2, sizeof(srcOffset), &srcOffset); - - // Program destinaiton origin - uint64_t dstOffset = dstOrigin[0] / CopyBuffAlignment[i]; - ; - setArgument(kernels_[blitType], 3, sizeof(dstOffset), &dstOffset); - - uint64_t copySize = size[0]; - setArgument(kernels_[blitType], 4, sizeof(copySize), ©Size); - - if (blitType == BlitCopyBufferAligned) { - int32_t alignment = CopyBuffAlignment[i]; - setArgument(kernels_[blitType], 5, sizeof(alignment), &alignment); - } else { - setArgument(kernels_[blitType], 5, sizeof(remain), &remain); - } - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(1, globalWorkOffset, &globalWorkSize, &localWorkSize); - - // Execute the blit - address parameters = kernels_[blitType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[blitType], parameters); - } else { - result = DmaBlitManager::copyBuffer(srcMemory, dstMemory, srcOrigin, dstOrigin, sizeIn, entire); - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::fillImage(device::Memory& memory, const void* pattern, - const amd::Coord3D& origin, const amd::Coord3D& size, - bool entire) const { - amd::ScopedLock k(lockXferOps_); - bool result = false; - - // Use host fill if memory has direct access - if (setup_.disableFillImage_ || gpuMem(memory).isHostMemDirectAccess()) { - result = HostBlitManager::fillImage(memory, pattern, origin, size, entire); - synchronize(); - return result; - } - - uint fillType; - size_t dim = 0; - size_t globalWorkOffset[3] = {0, 0, 0}; - size_t globalWorkSize[3]; - size_t localWorkSize[3]; - Memory* memView = &gpuMem(memory); - amd::Image::Format newFormat(gpuMem(memory).owner()->asImage()->getImageFormat()); - - // Program the kernels workload depending on the fill dimensions - fillType = FillImage; - dim = 3; - - void* newpattern = const_cast(pattern); - uint32_t iFillColor[4]; - - bool rejected = false; - bool releaseView = false; - // For depth, we need to create a view - if ((memView->cal()->format_ == CM_SURF_FMT_DEPTH32F) || - (memView->cal()->format_ == CM_SURF_FMT_RGBA8_SRGB) || - (memView->cal()->format_ == CM_SURF_FMT_DEPTH16)) { - // Find unsupported data type - for (uint i = 0; i < RejectedFormatDataTotal; ++i) { - if (RejectedData[i].clOldType_ == newFormat.image_channel_data_type) { - newFormat.image_channel_data_type = RejectedData[i].clNewType_; - rejected = true; - break; - } - } - - // Below may not be correct. We need to find why unsigned int view doesn't work for DEPTH16. - if (gpuMem(memory).cal()->format_ == CM_SURF_FMT_DEPTH16) { - newFormat.image_channel_data_type = CL_UNORM_INT16; - } - - if (gpuMem(memory).cal()->format_ == CM_SURF_FMT_RGBA8_SRGB) { - // Converting a linear RGB floating-point color value to a 8-bit unsigned integer sRGB value - // because hw is not support write_imagef for sRGB. - float* fColor = static_cast(newpattern); - iFillColor[0] = sRGBmap(fColor[0]); - iFillColor[1] = sRGBmap(fColor[1]); - iFillColor[2] = sRGBmap(fColor[2]); - iFillColor[3] = (uint32_t)(fColor[3] * 255.0f); - newpattern = static_cast(&iFillColor[0]); - for (uint i = 0; i < RejectedFormatChannelTotal; ++i) { - if (RejectedOrder[i].clOldType_ == newFormat.image_channel_order) { - newFormat.image_channel_order = RejectedOrder[i].clNewType_; - rejected = true; - break; - } - } - } - } - // If the image format was rejected, then attempt to create a view - if (rejected) { - memView = createView(gpuMem(memory), dev().getCalFormat(newFormat)); - if (memView != NULL) { - rejected = false; - releaseView = true; - } - } - - // Perform workload split to allow multiple operations in a single thread - globalWorkSize[0] = (size[0] + TransferSplitSize - 1) / TransferSplitSize; - // Find the current blit type - if (memView->cal()->dimSize_ == 1) { - globalWorkSize[0] = amd::alignUp(globalWorkSize[0], 256); - globalWorkSize[1] = amd::alignUp(size[1], 1); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = 256; - localWorkSize[1] = localWorkSize[2] = 1; - } else if (memView->cal()->dimSize_ == 2) { - globalWorkSize[0] = amd::alignUp(globalWorkSize[0], 16); - globalWorkSize[1] = amd::alignUp(size[1], 16); - globalWorkSize[2] = amd::alignUp(size[2], 1); - localWorkSize[0] = localWorkSize[1] = 16; - localWorkSize[2] = 1; - } else { - globalWorkSize[0] = amd::alignUp(globalWorkSize[0], 8); - globalWorkSize[1] = amd::alignUp(size[1], 8); - globalWorkSize[2] = amd::alignUp(size[2], 4); - localWorkSize[0] = localWorkSize[1] = 8; - localWorkSize[2] = 4; - } - - // Program kernels arguments for the blit operation - Memory* mem = memView; - setArgument(kernels_[fillType], 0, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 1, sizeof(float[4]), newpattern); - setArgument(kernels_[fillType], 2, sizeof(int32_t[4]), newpattern); - setArgument(kernels_[fillType], 3, sizeof(uint32_t[4]), newpattern); - - int32_t fillOrigin[4] = {(int32_t)origin[0], (int32_t)origin[1], (int32_t)origin[2], 0}; - int32_t fillSize[4] = {(int32_t)size[0], (int32_t)size[1], (int32_t)size[2], 0}; - setArgument(kernels_[fillType], 4, sizeof(fillOrigin), fillOrigin); - setArgument(kernels_[fillType], 5, sizeof(fillSize), fillSize); - - // Find the type of image - uint32_t type = 0; - switch (newFormat.image_channel_data_type) { - case CL_SNORM_INT8: - case CL_SNORM_INT16: - case CL_UNORM_INT8: - case CL_UNORM_INT16: - case CL_UNORM_SHORT_565: - case CL_UNORM_SHORT_555: - case CL_UNORM_INT_101010: - case CL_HALF_FLOAT: - case CL_FLOAT: - type = 0; - break; - case CL_SIGNED_INT8: - case CL_SIGNED_INT16: - case CL_SIGNED_INT32: - type = 1; - break; - case CL_UNSIGNED_INT8: - case CL_UNSIGNED_INT16: - case CL_UNSIGNED_INT32: - type = 2; - break; - } - setArgument(kernels_[fillType], 6, sizeof(type), &type); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(dim, globalWorkOffset, globalWorkSize, localWorkSize); - - // Execute the blit - address parameters = kernels_[fillType]->parameters().values(); - result = gpu().submitKernelInternal(ndrange, *kernels_[fillType], parameters); - if (releaseView) { - delete memView; - } - - synchronize(); - - return result; -} - -bool KernelBlitManager::runScheduler(device::Memory& vqueue, device::Memory& params, uint paramIdx, - uint threads) const { - amd::ScopedLock k(lockXferOps_); - - size_t globalWorkOffset[1] = {0}; - size_t globalWorkSize[1] = {threads}; - size_t localWorkSize[1] = {1}; - - // Program kernels arguments - Memory* q = &gpuMem(vqueue); - Memory* p = &gpuMem(params); - setArgument(kernels_[Scheduler], 0, sizeof(cl_mem), &q); - setArgument(kernels_[Scheduler], 1, sizeof(cl_mem), &p); - setArgument(kernels_[Scheduler], 2, sizeof(uint), ¶mIdx); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(1, globalWorkOffset, globalWorkSize, localWorkSize); - - // Execute the blit - address parameters = kernels_[Scheduler]->parameters().values(); - bool result = gpu().submitKernelInternal(ndrange, *kernels_[Scheduler], parameters); - - synchronize(); - - return result; -} - -amd::Memory* DmaBlitManager::pinHostMemory(const void* hostMem, size_t pinSize, - size_t& partial) const { - size_t pinAllocSize; - const static bool SysMem = true; - amd::Memory* amdMemory; - - // Allign offset to 4K boundary (Vista/Win7 limitation) - char* tmpHost = const_cast( - amd::alignDown(reinterpret_cast(hostMem), PinnedMemoryAlignment)); - - // Find the partial size for unaligned copy - partial = reinterpret_cast(hostMem) - tmpHost; - - // Recalculate pin memory size - pinAllocSize = amd::alignUp(pinSize + partial, PinnedMemoryAlignment); - - amdMemory = gpu().findPinnedMem(tmpHost, pinAllocSize); - - if (NULL != amdMemory) { - return amdMemory; - } - - amdMemory = new (*context_) amd::Buffer(*context_, CL_MEM_USE_HOST_PTR, pinAllocSize); - amdMemory->setVirtualDevice(&gpu()); - if ((amdMemory != NULL) && !amdMemory->create(tmpHost, SysMem)) { - amdMemory->release(); - return NULL; - } - - // Get device memory for this virtual device - // @note: This will force real memory pinning - Memory* srcMemory = dev().getGpuMemory(amdMemory); - - if (srcMemory == NULL) { - // Release all pinned memory and attempt pinning again - gpu().releasePinnedMem(); - srcMemory = dev().getGpuMemory(amdMemory); - if (srcMemory == NULL) { - // Release memory - amdMemory->release(); - amdMemory = NULL; - } - } - - return amdMemory; -} - -Memory* KernelBlitManager::createView(const Memory& parent, const CalFormat& format) const { - assert(!parent.cal()->buffer_ && "View supports images only"); - gpu::Memory* gpuImage = new gpu::Image( - dev(), parent.size(), parent.cal()->width_, parent.cal()->height_, - parent.cal()->depth_, format.type_, format.channelOrder_, - parent.cal()->imageType_, 1); - - // Create resource - if (NULL != gpuImage) { - Resource::ImageViewParams params; - const Memory& gpuMem = static_cast(parent); - - params.owner_ = parent.owner(); - params.level_ = 0; - params.layer_ = 0; - params.resource_ = &gpuMem; - params.memory_ = &gpuMem; - params.gpu_ = &gpu(); - - // Create memory object - bool result = gpuImage->create(Resource::ImageView, ¶ms); - if (!result) { - delete gpuImage; - return NULL; - } - } - - return gpuImage; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpublit.hpp b/projects/clr/rocclr/device/gpu/gpublit.hpp deleted file mode 100644 index 3dc9e977f9..0000000000 --- a/projects/clr/rocclr/device/gpu/gpublit.hpp +++ /dev/null @@ -1,427 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUBLIT_HPP_ -#define GPUBLIT_HPP_ - -#include "top.hpp" -#include "platform/command.hpp" -#include "device/gpu/gpudefs.hpp" -#include "device/device.hpp" -#include "device/blit.hpp" - -/*! \addtogroup GPU Blit Implementation - * @{ - */ - -//! GPU Blit Manager Implementation -namespace gpu { - -class Device; -class Kernel; -class Memory; -class VirtualGPU; - -//! DMA Blit Manager -class DmaBlitManager : public device::HostBlitManager { - public: - //! Constructor - DmaBlitManager(VirtualGPU& gpu, //!< Virtual GPU to be used for blits - Setup setup = Setup() //!< Specifies HW accelerated blits - ); - - //! Destructor - virtual ~DmaBlitManager() {} - - //! Creates DmaBlitManager object - virtual bool create(amd::Device& device) { return true; } - - //! Copies a buffer object to system memory - virtual bool readBuffer(device::Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destination host memory - const amd::Coord3D& origin, //!< Source origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to system memory - virtual bool readBufferRect(device::Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destinaiton host memory - const amd::BufferRect& bufRect, //!< Source rectangle - const amd::BufferRect& hostRect, //!< Destination rectangle - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies an image object to system memory - virtual bool readImage(device::Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destination host memory - const amd::Coord3D& origin, //!< Source origin - const amd::Coord3D& size, //!< Size of the copy region - size_t rowPitch, //!< Row pitch for host memory - size_t slicePitch, //!< Slice pitch for host memory - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies system memory to a buffer object - virtual bool writeBuffer(const void* srcHost, //!< Source host memory - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& origin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies system memory to a buffer object - virtual bool writeBufferRect(const void* srcHost, //!< Source host memory - device::Memory& dstMemory, //!< Destination memory object - const amd::BufferRect& hostRect, //!< Destination rectangle - const amd::BufferRect& bufRect, //!< Source rectangle - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies system memory to an image object - virtual bool writeImage(const void* srcHost, //!< Source host memory - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& origin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - size_t rowPitch, //!< Row pitch for host memory - size_t slicePitch, //!< Slice pitch for host memory - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to another buffer object - virtual bool copyBuffer(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to another buffer object - virtual bool copyBufferRect(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::BufferRect& srcRect, //!< Source rectangle - const amd::BufferRect& dstRect, //!< Destination rectangle - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies an image object to a buffer object - virtual bool copyImageToBuffer(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false, //!< Entire buffer will be updated - size_t rowPitch = 0, //!< Pitch for buffer - size_t slicePitch = 0 //!< Slice for buffer - ) const; - - //! Copies a buffer object to an image object - virtual bool copyBufferToImage(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false, //!< Entire buffer will be updated - size_t rowPitch = 0, //!< Pitch for buffer - size_t slicePitch = 0 //!< Slice for buffer - ) const; - - //! Copies an image object to another image object - virtual bool copyImage(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - protected: - static constexpr uint MaxPinnedBuffers = 4; - - //! Synchronizes the blit operations if necessary - inline void synchronize() const; - - //! Returns the virtual GPU object - VirtualGPU& gpu() const { return static_cast(vDev_); } - - //! Returns the GPU device object - const Device& dev() const { return static_cast(dev_); }; - - inline Memory& gpuMem(device::Memory& mem) const; - - //! Pins host memory for GPU access - amd::Memory* pinHostMemory(const void* hostMem, //!< Host memory pointer - size_t pinSize, //!< Host memory size - size_t& partial //!< Extra offset for memory alignment - ) const; - - const size_t MinSizeForPinnedTransfer; - bool completeOperation_; //!< DMA blit manager must complete operation - amd::Context* context_; //!< A dummy context - - private: - //! Disable copy constructor - DmaBlitManager(const DmaBlitManager&); - - //! Disable operator= - DmaBlitManager& operator=(const DmaBlitManager&); - - //! Reads video memory, using a staged buffer - bool readMemoryStaged(Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destination host memory - Memory** xferBuf, //!< Staged buffer for read - size_t origin, //!< Original offset in the source memory - size_t& offset, //!< Offset for the current copy pointer - size_t& totalSize, //!< Total size for copy region - size_t xferSize //!< Transfer size - ) const; - - //! Write into video memory, using a staged buffer - bool writeMemoryStaged(const void* srcHost, //!< Source host memory - Memory& dstMemory, //!< Destination memory object - Memory& xferBuf, //!< Staged buffer for write - size_t origin, //!< Original offset in the destination memory - size_t& offset, //!< Offset for the current copy pointer - size_t& totalSize, //!< Total size for the copy region - size_t xferSize //!< Transfer size - ) const; -}; - -//! Kernel Blit Manager -class KernelBlitManager : public DmaBlitManager { - public: - enum { - BlitCopyImage = 0, - BlitCopyImage1DA, - BlitCopyImageToBuffer, - BlitCopyBufferToImage, - BlitCopyBufferRect, - BlitCopyBufferRectAligned, - BlitCopyBuffer, - BlitCopyBufferAligned, - FillBufferAligned, - FillImage, - Scheduler, - BlitTotal - }; - - //! Constructor - KernelBlitManager(VirtualGPU& gpu, //!< Virtual GPU to be used for blits - Setup setup = Setup() //!< Specifies HW accelerated blits - ); - - //! Destructor - virtual ~KernelBlitManager(); - - //! Creates DmaBlitManager object - virtual bool create(amd::Device& device); - - //! Copies a buffer object to another buffer object - virtual bool copyBufferRect(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::BufferRect& srcRectIn, //!< Source rectangle - const amd::BufferRect& dstRectIn, //!< Destination rectangle - const amd::Coord3D& sizeIn, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to system memory - virtual bool readBuffer(device::Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destination host memory - const amd::Coord3D& origin, //!< Source origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to system memory - virtual bool readBufferRect(device::Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destinaiton host memory - const amd::BufferRect& bufRect, //!< Source rectangle - const amd::BufferRect& hostRect, //!< Destination rectangle - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies system memory to a buffer object - virtual bool writeBuffer(const void* srcHost, //!< Source host memory - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& origin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies system memory to a buffer object - virtual bool writeBufferRect(const void* srcHost, //!< Source host memory - device::Memory& dstMemory, //!< Destination memory object - const amd::BufferRect& hostRect, //!< Destination rectangle - const amd::BufferRect& bufRect, //!< Source rectangle - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to an image object - virtual bool copyBuffer(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies a buffer object to an image object - virtual bool copyBufferToImage(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false, //!< Entire buffer will be updated - size_t rowPitch = 0, //!< Pitch for buffer - size_t slicePitch = 0 //!< Slice for buffer - ) const; - - //! Copies an image object to a buffer object - virtual bool copyImageToBuffer(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false, //!< Entire buffer will be updated - size_t rowPitch = 0, //!< Pitch for buffer - size_t slicePitch = 0 //!< Slice for buffer - ) const; - - //! Copies an image object to another image object - virtual bool copyImage(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies an image object to system memory - virtual bool readImage(device::Memory& srcMemory, //!< Source memory object - void* dstHost, //!< Destination host memory - const amd::Coord3D& origin, //!< Source origin - const amd::Coord3D& size, //!< Size of the copy region - size_t rowPitch, //!< Row pitch for host memory - size_t slicePitch, //!< Slice pitch for host memory - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Copies system memory to an image object - virtual bool writeImage(const void* srcHost, //!< Source host memory - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& origin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - size_t rowPitch, //!< Row pitch for host memory - size_t slicePitch, //!< Slice pitch for host memory - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Fills a buffer memory with a pattern data - virtual bool fillBuffer(device::Memory& memory, //!< Memory object to fill with pattern - const void* pattern, //!< Pattern data - size_t patternSize, //!< Pattern size - const amd::Coord3D& origin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the fill region - bool entire = false, //!< Entire buffer will be updated - bool forceBlit = false //!< Force GPU Blit for fill - ) const; - - //! Fills an image memory with a pattern data - virtual bool fillImage(device::Memory& dstMemory, //!< Memory object to fill with pattern - const void* pattern, //!< Pattern data - const amd::Coord3D& origin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false //!< Entire buffer will be updated - ) const; - - //! Fills an image memory with a pattern data - virtual bool runScheduler(device::Memory& vqueue, //!< Memory object for virtual queue - device::Memory& params, //!< Extra arguments for the scheduler - uint paramIdx, //!< Parameter index - uint threads //!< Number of scheduling threads - ) const; - - private: - static constexpr size_t MaxXferBuffers = 2; - static constexpr uint TransferSplitSize = 3; - - //! Copies a buffer object to an image object - bool copyBufferToImageKernel(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false, //!< Entire buffer will be updated - size_t rowPitch = 0, //!< Pitch for buffer - size_t slicePitch = 0 //!< Slice for buffer - ) const; - - //! Copies an image object to a buffer object - bool copyImageToBufferKernel(device::Memory& srcMemory, //!< Source memory object - device::Memory& dstMemory, //!< Destination memory object - const amd::Coord3D& srcOrigin, //!< Source origin - const amd::Coord3D& dstOrigin, //!< Destination origin - const amd::Coord3D& size, //!< Size of the copy region - bool entire = false, //!< Entire buffer will be updated - size_t rowPitch = 0, //!< Pitch for buffer - size_t slicePitch = 0 //!< Slice for buffer - ) const; - - //! Creates a program for all blit operations - bool createProgram(Device& device //!< Device object - ); - - //! Creates a view memory object - Memory* createView(const Memory& parent, //!< Parent memory object - const CalFormat& format //!< The new format for a view - ) const; - - //! Disable copy constructor - KernelBlitManager(const KernelBlitManager&); - - //! Disable operator= - KernelBlitManager& operator=(const KernelBlitManager&); - - amd::Program* program_; //!< GPU program object - amd::Kernel* kernels_[BlitTotal]; //!< GPU kernels for blit - amd::Memory* constantBuffer_; //!< An internal CB for blits - amd::Memory* xferBuffers_[MaxXferBuffers]; //!< Transfer buffers for images - size_t xferBufferSize_; //!< Transfer buffer size - amd::Monitor* lockXferOps_; //!< Lock transfer operation -}; - -static const char* BlitName[KernelBlitManager::BlitTotal] = { - "__amd_rocclr_copyImage", "__amd_rocclr_copyImage1DA", "__amd_rocclr_copyImageToBuffer", - "__amd_rocclr_copyBufferToImage", "__amd_rocclr_copyBufferRect", - "__amd_rocclr_copyBufferRectAligned", "__amd_rocclr_copyBuffer", - "__amd_rocclr_copyBufferAligned", "__amd_rocclr_fillBufferAligned", - "__amd_rocclr_fillImage", "__amd_rocclr_scheduler" -}; - -/*@}*/} // namespace gpu - -#endif /*GPUBLIT_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpucompiler.cpp b/projects/clr/rocclr/device/gpu/gpucompiler.cpp deleted file mode 100644 index 540620350d..0000000000 --- a/projects/clr/rocclr/device/gpu/gpucompiler.cpp +++ /dev/null @@ -1,348 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include -#include -#include -#include - -#include "os/os.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpukernel.hpp" -#include "utils/options.hpp" -#include - -// CLC_IN_PROCESS_CHANGE -extern int openclFrontEnd(const char* cmdline, std::string*, std::string* typeInfo = NULL); - -namespace gpu { - -static int programsCount = 0; - -bool NullProgram::compileImpl(const std::string& src, - const std::vector& headers, - const char** headerIncludeNames, amd::option::Options* options) { - std::string sourceCode = src; - - if (gpuNullDevice().settings().debugFlags_ & Settings::CheckForILSource) { - size_t inc = sourceCode.find("il_cs_", 0); - if (inc != std::string::npos) { - // CL program is an IL program - ilProgram_ = sourceCode; - return true; - } - } - - - std::string tempFolder = amd::Os::getTempPath(); - std::string tempFileName = amd::Os::getTempFileName(); - - if (gpuNullDevice().settings().debugFlags_ & Settings::StubCLPrograms) { - std::stringstream fileName; - std::fstream stubRead; - // Dump the IL function - fileName << "program_" << programsCount++ << ".cl"; - stubRead.open(fileName.str().c_str(), (std::fstream::in | std::fstream::binary)); - // Check if we have OpenCL program - if (stubRead.is_open()) { - // Find the stream size - stubRead.seekg(0, std::fstream::end); - size_t size = stubRead.tellg(); - stubRead.seekg(0, std::ios::beg); - - char* data = new char[size]; - stubRead.read(data, size); - stubRead.close(); - - sourceCode.assign(data, size); - delete[] data; - } else { - std::fstream stubWrite; - stubWrite.open(fileName.str().c_str(), (std::fstream::out | std::fstream::binary)); - stubWrite << sourceCode; - stubWrite.close(); - } - } - - std::fstream f; - std::vector headerFileNames(headers.size()); - std::vector newDirs; - for (size_t i = 0; i < headers.size(); ++i) { - std::string headerPath = tempFolder; - std::string headerIncludeName(headerIncludeNames[i]); - // replace / in path with current os's file separator - if (amd::Os::fileSeparator() != '/') { - for (auto& it : headerIncludeName) { - if (it == '/') it = amd::Os::fileSeparator(); - } - } - size_t pos = headerIncludeName.rfind(amd::Os::fileSeparator()); - if (pos != std::string::npos) { - headerPath += amd::Os::fileSeparator(); - headerPath += headerIncludeName.substr(0, pos); - headerIncludeName = headerIncludeName.substr(pos + 1); - } - if (!amd::Os::pathExists(headerPath)) { - bool ret = amd::Os::createPath(headerPath); - assert(ret && "failed creating path!"); - newDirs.push_back(headerPath); - } - std::string headerFullName = headerPath + amd::Os::fileSeparator() + headerIncludeName; - headerFileNames[i] = headerFullName; - f.open(headerFullName.c_str(), std::fstream::out); - assert(!f.fail() && "failed creating header file!"); - f.write(headers[i]->c_str(), headers[i]->length()); - f.close(); - } - - acl_error err; - const aclTargetInfo& targInfo = info(); - - aclBinaryOptions binOpts = {0}; - binOpts.struct_size = sizeof(binOpts); - binOpts.elfclass = targInfo.arch_id == aclAMDIL64 ? ELFCLASS64 : ELFCLASS32; - binOpts.bitness = ELFDATA2LSB; - binOpts.alloc = &::malloc; - binOpts.dealloc = &::free; - - aclBinary* bin = amd::Hsail::BinaryInit(sizeof(aclBinary), &targInfo, &binOpts, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclBinaryInit failed"); - return false; - } - - if (ACL_SUCCESS != - amd::Hsail::InsertSection(gpuNullDevice().amdilCompiler(), bin, sourceCode.c_str(), sourceCode.size(), aclSOURCE)) { - LogWarning("aclInsertSection failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - - // temporary solution to synchronize buildNo between runtime and complib - // until we move runtime inside complib - ((amd::option::Options*)bin->options)->setBuildNo(options->getBuildNo()); - - std::stringstream opts; - std::string token; - opts << options->origOptionStr.c_str(); - - if (options->origOptionStr.find("-cl-std=CL") == std::string::npos) { - switch (gpuNullDevice().settings().oclVersion_) { - case OpenCL10: - opts << " -cl-std=CL1.0"; - break; - case OpenCL11: - opts << " -cl-std=CL1.1"; - break; - case OpenCL20: - case OpenCL21: - default: - case OpenCL12: - opts << " -cl-std=CL1.2"; - break; - } - } - - // FIXME: Should we prefix everything with -Wf,? - std::istringstream iss(options->clcOptions); - while (getline(iss, token, ' ')) { - if (!token.empty()) { - // Check if this is a -D option - if (token.compare("-D") == 0) { - // It is, skip payload - getline(iss, token, ' '); - continue; - } - opts << " -Wf," << token; - } - } - - if (!headers.empty()) { - opts << " -I" << tempFolder; - } - - if (!gpuNullDevice().settings().imageSupport_) { - opts << " -fno-image-support"; - } - - if (gpuNullDevice().settings().reportFMAF_) { - opts << " -mfast-fmaf"; - } - - if (gpuNullDevice().settings().reportFMA_) { - opts << " -mfast-fma"; - } - - iss.clear(); - iss.str(device().info().extensions_); - while (getline(iss, token, ' ')) { - if (!token.empty()) { - opts << " -D" << token << "=1"; - } - } - - std::string newOpt = opts.str(); - size_t pos = newOpt.find("-fno-bin-llvmir"); - while (pos != std::string::npos) { - newOpt.erase(pos, 15); - pos = newOpt.find("-fno-bin-llvmir"); - } - - err = amd::Hsail::Compile(gpuNullDevice().amdilCompiler(), bin, newOpt.c_str(), ACL_TYPE_OPENCL, ACL_TYPE_LLVMIR_BINARY, - NULL); - - buildLog_ += amd::Hsail::GetCompilerLog(gpuNullDevice().amdilCompiler()); - - if (err != ACL_SUCCESS) { - LogWarning("aclCompile failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - - size_t len = 0; - const void* ir = amd::Hsail::ExtractSection(gpuNullDevice().amdilCompiler(), bin, &len, aclLLVMIR, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclExtractSection failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - - llvmBinary_.assign(reinterpret_cast(ir), len); - elfSectionType_ = amd::Elf::LLVMIR; - amd::Hsail::BinaryFini(bin); - - for (size_t i = 0; i < headerFileNames.size(); ++i) { - amd::Os::unlink(headerFileNames[i].c_str()); - } - for (size_t i = 0; i < newDirs.size(); ++i) { - amd::Os::removePath(newDirs[i]); - } - -#ifdef _WIN32 - amd::Os::unlink(tempFileName); -#endif - - if (clBinary()->saveSOURCE()) { - clBinary()->elfOut()->addSection(amd::Elf::SOURCE, sourceCode.data(), sourceCode.size()); - } - if (clBinary()->saveLLVMIR()) { - clBinary()->elfOut()->addSection(amd::Elf::LLVMIR, llvmBinary_.data(), llvmBinary_.size()); - // store the original compile options - clBinary()->storeCompileOptions(compileOptions_); - } - - return true; -} - -int NullProgram::compileBinaryToIL(amd::option::Options* options) { - acl_error err; - const aclTargetInfo& targInfo = info(); - - aclBinaryOptions binOpts = {0}; - binOpts.struct_size = sizeof(binOpts); - binOpts.elfclass = targInfo.arch_id == aclAMDIL64 ? ELFCLASS64 : ELFCLASS32; - binOpts.bitness = ELFDATA2LSB; - binOpts.alloc = &::malloc; - binOpts.dealloc = &::free; - - aclBinary* bin = amd::Hsail::BinaryInit(sizeof(aclBinary), &targInfo, &binOpts, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclBinaryInit failed"); - return CL_BUILD_PROGRAM_FAILURE; - } - aclSections_0_8 spirFlag; - _acl_type_enum_0_8 aclTypeBinaryUsed; - if (std::string::npos != options->clcOptions.find("--spirv") || - elfSectionType_ == amd::Elf::SPIRV) { - spirFlag = aclSPIRV; - aclTypeBinaryUsed = ACL_TYPE_SPIRV_BINARY; - } else if (std::string::npos != options->clcOptions.find("--spir") || - elfSectionType_ == amd::Elf::SPIR) { - spirFlag = aclSPIR; - aclTypeBinaryUsed = ACL_TYPE_SPIR_BINARY; - } else { - spirFlag = aclLLVMIR; - aclTypeBinaryUsed = ACL_TYPE_LLVMIR_BINARY; - } - - if (ACL_SUCCESS != - amd::Hsail::InsertSection(gpuNullDevice().amdilCompiler(), bin, llvmBinary_.data(), llvmBinary_.size(), spirFlag)) { - LogWarning("aclInsertSection failed"); - amd::Hsail::BinaryFini(bin); - return CL_BUILD_PROGRAM_FAILURE; - } - - // pass kernel argument alignment info to compiler lib through option str - std::string optionStr = options->origOptionStr; - if (options->origOptionStr.find("kernel-arg-alignment") == std::string::npos) { - char s[256]; - sprintf(s, " -Wb,-kernel-arg-alignment=%d", gpuNullDevice().info().memBaseAddrAlign_ / 8); - optionStr += s; - } - - // temporary solution to synchronize buildNo between runtime and complib - // until we move runtime inside complib - ((amd::option::Options*)bin->options)->setBuildNo(options->getBuildNo()); - - aclType type = ACL_TYPE_CG; - // If option bin-bif30 is set, generate BIF 3.0 binary - if (options->oVariables->BinBIF30) { - type = ACL_TYPE_ISA; - } - - err = amd::Hsail::Compile(gpuNullDevice().amdilCompiler(), bin, optionStr.c_str(), aclTypeBinaryUsed, type, NULL); - buildLog_ += amd::Hsail::GetCompilerLog(gpuNullDevice().amdilCompiler()); - - if (err != ACL_SUCCESS) { - LogWarning("aclCompile failed"); - amd::Hsail::BinaryFini(bin); - return CL_BUILD_PROGRAM_FAILURE; - } - - if (options->oVariables->BinBIF30) { - acl_error err; - char* binaryIn = nullptr; - size_t size; - err = amd::Hsail::WriteToMem(bin, reinterpret_cast(&binaryIn), &size); - if (err != ACL_SUCCESS) { - LogWarning("aclWriteToMem failed"); - amd::Hsail::BinaryFini(bin); - return CL_BUILD_PROGRAM_FAILURE; - } - clBinary()->saveBIFBinary(binaryIn, size); - amd::Hsail::FreeMem(bin, binaryIn); - } - - size_t len = 0; - const void* amdil = amd::Hsail::ExtractSection(gpuNullDevice().amdilCompiler(), bin, &len, aclCODEGEN, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclExtractSection failed"); - amd::Hsail::BinaryFini(bin); - return CL_BUILD_PROGRAM_FAILURE; - } - - ilProgram_.assign(reinterpret_cast(amdil), len); - amd::Hsail::BinaryFini(bin); - - return CL_SUCCESS; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuconstbuf.cpp b/projects/clr/rocclr/device/gpu/gpuconstbuf.cpp deleted file mode 100644 index 03039d473d..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuconstbuf.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/gpu/gpuconstbuf.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpusettings.hpp" - -namespace gpu { - -ConstBuffer::ConstBuffer(VirtualGPU& gpu, size_t size) - : Memory(const_cast(gpu.dev()), size * VectorSize), - gpu_(gpu), - size_(size * VectorSize), - wrtOffset_(0), - lastWrtSize_(0), - wrtAddress_(NULL) {} - -ConstBuffer::~ConstBuffer() { - if (wrtAddress_ != NULL) { - unmap(&gpu_); - } - - amd::AlignedMemory::deallocate(sysMemCopy_); -} - -bool ConstBuffer::create() { - // Create sysmem copy for the constant buffer - sysMemCopy_ = reinterpret_cast
(amd::AlignedMemory::allocate(size_, 256)); - if (sysMemCopy_ == NULL) { - LogPrintfError( - "We couldn't allocate sysmem copy for constant buffer,\ - size(%d)!", - size_); - return false; - } - memset(sysMemCopy_, 0, size_); - - if (!Memory::create(Resource::RemoteUSWC)) { - LogPrintfError("We couldn't create HW constant buffer, size(%d)!", size_); - return false; - } - - // Constant buffer warm-up - warmUpRenames(gpu_); - - wrtAddress_ = map(&gpu_, Resource::Discard); - if (wrtAddress_ == NULL) { - LogPrintfError("We couldn't map HW constant buffer, size(%d)!", size_); - return false; - } - - return true; -} - -bool ConstBuffer::uploadDataToHw(size_t size) { - static const size_t HwCbAlignment = 256; - - // Align copy size on the vector's boundary - size_t count = amd::alignUp(size, VectorSize); - wrtOffset_ += lastWrtSize_; - - // Check if CB has enough space for copy - if ((wrtOffset_ + count) > size_) { - if (wrtAddress_ != NULL) { - unmap(&gpu_); - } - wrtAddress_ = map(&gpu_, Resource::Discard); - wrtOffset_ = 0; - lastWrtSize_ = 0; - } - - // Update memory with new CB data - memcpy((reinterpret_cast(wrtAddress_) + wrtOffset_), sysMemCopy_, count); - - // Adjust the size by the HW CB buffer alignment - lastWrtSize_ = amd::alignUp(size, HwCbAlignment); - return true; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuconstbuf.hpp b/projects/clr/rocclr/device/gpu/gpuconstbuf.hpp deleted file mode 100644 index 83a3adad58..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuconstbuf.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUCONSTBUF_HPP_ -#define GPUCONSTBUF_HPP_ - -#include "device/gpu/gpumemory.hpp" - -//! \namespace gpu GPU Resource Implementation -namespace gpu { - -//! Cconstant buffer -class ConstBuffer : public Memory { - public: - //! Vector size of the constant buffer - static constexpr size_t VectorSize = 16; - - //! Constructor for the ConstBuffer class - ConstBuffer(VirtualGPU& gpu, //!< Virtual GPU device object - size_t size //!< size of the constant buffer in vectors - ); - - //! Destructor for the ConstBuffer class - ~ConstBuffer(); - - //! Creates the real HW constant buffer - bool create(); - - /*! \brief Uploads current constant buffer data from sysMemCopy_ to HW - * - * \return True if the data upload was succesful - */ - bool uploadDataToHw(size_t size //!< real data size for upload - ); - - //! Returns a pointer to the system memory copy for CB - address sysMemCopy() const { return sysMemCopy_; } - - //! Returns CB size - size_t size() const { return size_; } - - //! Returns current write offset for the constant buffer - size_t wrtOffset() const { return wrtOffset_; } - - //! Returns last write size for the constant buffer - size_t lastWrtSize() const { return lastWrtSize_; } - - private: - //! Disable copy constructor - ConstBuffer(const ConstBuffer&); - - //! Disable operator= - ConstBuffer& operator=(const ConstBuffer&); - - VirtualGPU& gpu_; //!< Virtual GPU object - address sysMemCopy_; //!< System memory copy - size_t size_; //!< Constant buffer size - size_t wrtOffset_; //!< Current write offset - size_t lastWrtSize_; //!< Last write size - void* wrtAddress_; //!< Write address in CB -}; - - -/*@}*/} // namespace gpu - -#endif /*GPUCONSTBUF_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpucounters.cpp b/projects/clr/rocclr/device/gpu/gpucounters.cpp deleted file mode 100644 index 95e77dc2ef..0000000000 --- a/projects/clr/rocclr/device/gpu/gpucounters.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpucounters.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "query/PerformanceQueryObject.h" - -namespace gpu { - -CalCounterReference::~CalCounterReference() { - // The counter object is always associated with a particular queue, - // so we have to lock just this queue - amd::ScopedLock lock(gpu_.execution()); - - if (0 != counter_) { - gpu().cs()->destroyQuery(gslCounter()); - } -} - -bool CalCounterReference::growResultArray(uint index) { - if (results_ != NULL) { - delete[] results_; - } - results_ = new uint64_t[index + 1]; - if (results_ == NULL) { - return false; - } - return true; -} - -PerfCounter::~PerfCounter() { - if (calRef_ == NULL) { - return; - } - - // Release the counter reference object - calRef_->release(); -} - -bool PerfCounter::create(CalCounterReference* calRef) { - assert(&gpu() == &calRef->gpu()); - - calRef_ = calRef; - counter_ = calRef->gslCounter(); - index_ = calRef->retain() - 2; - calRef->growResultArray(index_); - - // Initialize the counter - gslCounter()->getAsPerformanceQueryObject()->setCounterState( - info()->blockIndex_, info()->counterIndex_, info()->eventIndex_); - - return true; -} - -uint64_t PerfCounter::getInfo(uint64_t infoType) const { - switch (infoType) { - case CL_PERFCOUNTER_GPU_BLOCK_INDEX: { - // Return the GPU block index - return info()->blockIndex_; - } - case CL_PERFCOUNTER_GPU_COUNTER_INDEX: { - // Return the GPU counter index - return info()->counterIndex_; - } - case CL_PERFCOUNTER_GPU_EVENT_INDEX: { - // Return the GPU event index - return info()->eventIndex_; - } - case CL_PERFCOUNTER_DATA: { - gslCounter()->GetResult(gpu().cs(), reinterpret_cast(calRef_->results())); - return calRef_->results()[index_]; - } - default: - LogError("Wrong PerfCounter::getInfo parameter"); - } - return 0; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpucounters.hpp b/projects/clr/rocclr/device/gpu/gpucounters.hpp deleted file mode 100644 index f0500008ff..0000000000 --- a/projects/clr/rocclr/device/gpu/gpucounters.hpp +++ /dev/null @@ -1,140 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUCOUNTERS_HPP_ -#define GPUCOUNTERS_HPP_ - -#include "top.hpp" -#include "device/device.hpp" -#include "device/gpu/gpudevice.hpp" - -namespace gpu { - -class VirtualGPU; - -class CalCounterReference : public amd::ReferenceCountedObject { - public: - //! Default constructor - CalCounterReference(VirtualGPU& gpu, //!< Virtual GPU device object - gslQueryObject gslCounter) - : gpu_(gpu), counter_(gslCounter), results_(NULL) {} - - //! Get CAL counter - gslQueryObject gslCounter() const { return counter_; } - - //! Returns the virtual GPU device - const VirtualGPU& gpu() const { return gpu_; } - - //! Increases the results array for this CAL counter(container) - bool growResultArray(uint maxIndex //!< the maximum HW counter index in the CAL counter - ); - - //! Returns the CAL counter results - uint64_t* results() const { return results_; } - - protected: - //! Default destructor - ~CalCounterReference(); - - private: - //! Disable copy constructor - CalCounterReference(const CalCounterReference&); - - //! Disable operator= - CalCounterReference& operator=(const CalCounterReference&); - - VirtualGPU& gpu_; //!< The virtual GPU device object - gslQueryObject counter_; //!< GSL object counter - uint64_t* results_; //!< CAL counter results -}; - -//! Performance counter implementation on GPU -class PerfCounter : public device::PerfCounter { - public: - //! The performance counter info - struct Info : public amd::EmbeddedObject { - uint blockIndex_; //!< Index of the block to configure - uint counterIndex_; //!< Index of the hardware counter - uint eventIndex_; //!< Event you wish to count with the counter - }; - - //! The PerfCounter flags - enum Flags { BeginIssued = 0x00000001, EndIssued = 0x00000002, ResultReady = 0x00000004 }; - - //! Constructor for the GPU PerfCounter object - PerfCounter(const Device& device, //!< A GPU device object - const VirtualGPU& gpu, //!< Virtual GPU device object - uint32_t blockIndex, //!< HW block index - uint32_t counterIndex, //!< Counter index within the block - uint32_t eventIndex) //!< Event index for profiling - : gpuDevice_(device), - gpu_(gpu), - calRef_(NULL), - flags_(0), - counter_(0), - index_(0) { - info_.blockIndex_ = blockIndex; - info_.counterIndex_ = counterIndex; - info_.eventIndex_ = eventIndex; - } - - //! Destructor for the GPU PerfCounter object - virtual ~PerfCounter(); - - //! Creates the current object - bool create(CalCounterReference* calRef //!< Reference counter - ); - - //! Returns the specific information about the counter - uint64_t getInfo(uint64_t infoType //!< The type of returned information - ) const; - - //! Returns the GPU device, associated with the current object - const Device& dev() const { return gpuDevice_; } - - //! Returns the virtual GPU device - const VirtualGPU& gpu() const { return gpu_; } - - //! Returns the CAL performance counter descriptor - const Info* info() const { return &info_; } - - //! Returns the Info structure for performance counter - gslQueryObject gslCounter() const { return counter_; } - - private: - //! Disable default copy constructor - PerfCounter(const PerfCounter&); - - //! Disable default operator= - PerfCounter& operator=(const PerfCounter&); - - const Device& gpuDevice_; //!< The backend device - const VirtualGPU& gpu_; //!< The virtual GPU device object - - CalCounterReference* calRef_; //!< Reference counter - uint flags_; //!< The perfcounter object state - Info info_; //!< The info structure for perfcounter - gslQueryObject counter_; //!< GSL counter object - uint index_; //!< Counter index in the CAL container -}; - -} // namespace gpu - -#endif // GPUCOUNTERS_HPP_ diff --git a/projects/clr/rocclr/device/gpu/gpudebugger.hpp b/projects/clr/rocclr/device/gpu/gpudebugger.hpp deleted file mode 100644 index e375463550..0000000000 --- a/projects/clr/rocclr/device/gpu/gpudebugger.hpp +++ /dev/null @@ -1,133 +0,0 @@ -/* Copyright (c) 2014 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef HWDBG_GPUDEBGGER_H_ -#define HWDBG_GPUDEBGGER_H_ - -#include -#include -#include "hsa.h" -#include "amd_hsa_kernel_code.h" -#include "device/device.hpp" -#include "device/hwdebug.hpp" -#include "hsailctx.hpp" - -static constexpr int NumberReserveVgprs = 4; - -namespace gpu { - -/** - * \defgroup Services_API OCL Runtime Services API - * @{ - */ - - -/*! \brief Dispatch packet information - * - * This structure contains the packet information for kernel dispatch - */ -struct PacketAmdInfo { - uint32_t trapReservedVgprIndex_; //!< reserved VGPR index, -1 when they are not valid - uint32_t scratchBufferWaveOffset_; //!< scratch buffer wave offset, -1 when no scratch buffer - void* pointerToIsaBuffer_; //!< pointer to the buffer containing ISA - size_t sizeOfIsaBuffer_; //!< size of the ISA buffer - uint32_t numberOfVgprs_; //!< number of VGPRs used by the kernel - uint32_t numberOfSgprs_; //!< number of SGPRs used by the kernel - size_t sizeOfStaticGroupMemory_; //!< Static local memory used by the kernel -}; - -/*! \brief Cache mask for invalidation - */ -struct HwDbgGpuCacheMask { - HwDbgGpuCacheMask() : ui32All_(0) {} - - HwDbgGpuCacheMask(uint32_t mask) : ui32All_(mask) {} - - union { - struct { - uint32_t sqICache_ : 1; //!< Instruction cache - uint32_t sqKCache_ : 1; //!< Data cache - uint32_t tcL1_ : 1; //!< tcL1 cache - uint32_t tcL2_ : 1; //!< tcL2 cache - uint32_t reserved_ : 28; - }; - uint32_t ui32All_; - }; -}; - -/*! \brief Address watch information - * - * Information about each watch point - address, mask, mode and event - */ -struct HwDbgAddressWatch { - void* watchAddress_; //! The address of watch point - uint64_t watchMask_; //! The mask for watch point (lower 24 bits) - cl_dbg_address_watch_mode_amd watchMode_; //! The watch mode for this watch - DebugEvent event_; //! Event of the watch point (not used for now) -}; - -/*! \brief Runtime structure used to communicate debug information - * between Ocl services and core for a kernel dispatch. - */ -struct DebugToolInfo { - uint64_t scratchAddress_; //! Scratch memory address - size_t scratchSize_; //! Scratch memory size - uint64_t globalAddress_; //! Global memory address - uint32_t cacheDisableMask_; //! Cache mask, indicating caches disabled - uint32_t exceptionMask_; //! Exception mask - uint32_t reservedCuNum_; //! Number of reserved CUs for display, - //! which ranges from 0 to 7 in the current implementation. - bool monitorMode_; //! Debug or profiler mode - bool gpuSingleStepMode_; //! SQ debug mode - amd::Memory* trapHandler_; //! Trap handler address - amd::Memory* trapBuffer_; //! Trap buffer address - bool sqPerfcounterEnable_; //! whether SQ perf counters are enabled - aclBinary* aclBinary_; //! pointer of the kernel ACL binary - amd::Event* event_; //! pointer of the kernel event in the enqueue command -}; - -/*! \brief Message used by the KFD wave control for CI - * - * Structure indicates the various information used by the wave control function. - */ -struct HwDebugWaveAddr { - uint32_t VMID_ : 4; //! Virtual memory id - uint32_t wave_ : 4; //! Wave id - uint32_t SIMD_ : 2; //! SIMD id - uint32_t CU_ : 4; //! Compute unit - uint32_t SH_ : 1; //! Shader array - uint32_t SE_ : 1; //! Shader engine -}; - -/*! \brief Kernel code information -* -* This structure contains the pointer of mapped kernel code for host access -* and its size (in bytes) -*/ -struct AqlCodeInfo { - amd_kernel_code_t* aqlCode_; //! pointer of AQL code to allow host access - uint32_t aqlCodeSize_; //! size of AQL code -}; - -/**@}*/ - -} // namespace gpu - -#endif // HWDBG_GPUDEBGGER_H_ diff --git a/projects/clr/rocclr/device/gpu/gpudebugmanager.cpp b/projects/clr/rocclr/device/gpu/gpudebugmanager.cpp deleted file mode 100644 index 39d5a99df4..0000000000 --- a/projects/clr/rocclr/device/gpu/gpudebugmanager.cpp +++ /dev/null @@ -1,354 +0,0 @@ -/* Copyright (c) 2014 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gpudebugmanager.hpp" -#include "gpudevice.hpp" -#include "platform/commandqueue.hpp" - -#include "device/device.hpp" -#include "device/gpu/gpumemory.hpp" -#include "device/gpu/gputrap.hpp" -#include -#include -#include - -namespace gpu { - -class VirtualGPU; -class Device; -class Memory; - -/* - *************************************************************************** - * Implementation of GPU Debug Manager class - *************************************************************************** - */ - -GpuDebugManager::GpuDebugManager(amd::Device* device) - : HwDebugManager(device), - vGpu_(NULL), - debugMessages_(0), - addressWatch_(NULL), - addressWatchSize_(0), - oclEventHandle_(NULL) { - // Initialize the exception info and the kernel execution mode - excpPolicy_.exceptionMask = 0x0; - excpPolicy_.waveAction = CL_DBG_WAVES_RESUME; - excpPolicy_.hostAction = CL_DBG_HOST_IGNORE; - excpPolicy_.waveMode = CL_DBG_WAVEMODE_BROADCAST; - - execMode_.ui32All = 0; - - rtTrapHandlerInfo_.trap_.trapHandler_ = NULL; - rtTrapHandlerInfo_.trap_.trapBuffer_ = NULL; - - aqlPacket_ = (hsa_kernel_dispatch_packet_t*)NULL; - - return; -} - -GpuDebugManager::~GpuDebugManager() { - if (NULL != addressWatch_) { - delete[] addressWatch_; - } -} - -void GpuDebugManager::executePreDispatchCallBack(void* aqlPacket, void* toolInfo) { - DebugToolInfo* info = reinterpret_cast(toolInfo); - - aqlPacket_ = reinterpret_cast(aqlPacket); - - // Only if the pre-dispatch callback is set, will we update cache - // flush configuration and build the memory descriptor. - if (NULL != preDispatchCallBackFunc_) { - // Build the scratch memory descriptor - device()->gslCtx()->BuildScratchBufferResource(debugInfo_.scratchMemoryDescriptor_, - info->scratchAddress_, info->scratchSize_); - - // Build the global memory descriptor - device()->gslCtx()->BuildHeapBufferResource(debugInfo_.globalMemoryDescriptor_, - info->globalAddress_); - - // // for invalidate cache (BuildEndOfKernelNotifyCommands) - // aqlPacket->release_fence_scope = 2; - - aclBinary_ = reinterpret_cast(info->aclBinary_); - oclEventHandle_ = reinterpret_cast(as_cl(info->event_)); - - cl_device_id clDeviceId = as_cl(device_); - preDispatchCallBackFunc_(clDeviceId, oclEventHandle_, aqlPacket_, aclBinary_, - preDispatchCallBackArgs_); - } - - // setup the trap handler information only if the debugger has been registered - if (isRegistered()) { - // Copy the various info set by the debugger/profiler to the tool info structure - setupTrapInformation(info); - } -} - -void GpuDebugManager::executePostDispatchCallBack() { - if (NULL != postDispatchCallBackFunc_) { - cl_device_id clDeviceId = as_cl(device_); - postDispatchCallBackFunc_(clDeviceId, aqlPacket_->completion_signal.handle, - postDispatchCallBackArgs_); - } -} - -//! Map the kernel code for host access -void GpuDebugManager::mapKernelCode(void* aqlCodeInfo) const { - AqlCodeInfo* codeInfo = reinterpret_cast(aqlCodeInfo); - - codeInfo->aqlCode_ = reinterpret_cast(aqlCodeAddr_); - codeInfo->aqlCodeSize_ = aqlCodeSize_; -} - -int32_t GpuDebugManager::registerDebugger(amd::Context* context, uintptr_t messageStorage) { - if (!device()->settings().enableHwDebug_) { - LogError("debugmanager: Register debugger error - HW DEBUG is not enable"); - return CL_DEBUGGER_REGISTER_FAILURE_AMD; - } - - // first time register - set the message storage, flush queue and enable hw debug - if (!isRegistered()) { - debugMessages_ = messageStorage; - if (!device()->gslCtx()->registerHwDebugger(debugMessages_)) { - LogError("debugmanager: Register debugger failed"); - return CL_OUT_OF_RESOURCES; - } - - isRegistered_ = true; - - if (CL_SUCCESS != createRuntimeTrapHandler()) { - LogError("debugmanager: Create runtime trap handler failed"); - return CL_OUT_OF_RESOURCES; - } - } - - context_ = context; - - return CL_SUCCESS; -} - -void GpuDebugManager::unregisterDebugger() { - if (isRegistered()) { - // reset the debugger registration flag - isRegistered_ = false; - context_ = NULL; - } -} - -void GpuDebugManager::flushCache(uint32_t mask) { - HwDbgGpuCacheMask cacheMask(mask); - device()->xferQueue()->flushCuCaches(cacheMask); -} - - -void GpuDebugManager::setupTrapInformation(DebugToolInfo* toolInfo) { - toolInfo->scratchAddress_ = 0; - toolInfo->scratchSize_ = 0; - toolInfo->globalAddress_ = 0; - toolInfo->sqPerfcounterEnable_ = false; - - // Set up trap related info in the kernel info structure to be - // used in the kernel dispatch. - toolInfo->exceptionMask_ = excpPolicy_.exceptionMask; - toolInfo->gpuSingleStepMode_ = execMode_.gpuSingleStepMode; - toolInfo->monitorMode_ = execMode_.monitorMode; - - // The order of these three bits is determined by the definition - // of the register COMPUTE_DISPATCH_INITIATOR - toolInfo->cacheDisableMask_ = ((execMode_.disableL1Scalar << 2) | - (execMode_.disableL2Cache << 1) | (execMode_.disableL1Vector)); - - toolInfo->reservedCuNum_ = execMode_.reservedCuNum; - - toolInfo->trapHandler_ = rtTrapInfo_[kDebugTrapHandlerLocation]; - toolInfo->trapBuffer_ = rtTrapInfo_[kDebugTrapBufferLocation]; -} - -void GpuDebugManager::getPacketAmdInfo(const void* aqlCodeInfo, void* packetInfo) const - -{ - const AqlCodeInfo* codeInfo = reinterpret_cast(aqlCodeInfo); - - const amd_kernel_code_t* hostAqlCode = codeInfo->aqlCode_; - - PacketAmdInfo* packet = reinterpret_cast(packetInfo); - - const amd_kernel_code_t* akc = hostAqlCode; - - packet->numberOfSgprs_ = akc->wavefront_sgpr_count; - packet->numberOfVgprs_ = akc->workitem_vgpr_count; - - // use mapped kernel_object_address for host accessing of ISA buffer - packet->pointerToIsaBuffer_ = (char*)(hostAqlCode) + akc->kernel_code_entry_byte_offset; - - packet->scratchBufferWaveOffset_ = akc->debug_wavefront_private_segment_offset_sgpr; - - packet->sizeOfIsaBuffer_ = codeInfo->aqlCodeSize_; - - packet->sizeOfStaticGroupMemory_ = akc->workgroup_group_segment_byte_size; - - // The trap_reserved_vgpr_index will be 4 less the original - // This value must be used only by the debugger - packet->trapReservedVgprIndex_ = akc->workitem_vgpr_count - NumberReserveVgprs; -} - -DebugEvent GpuDebugManager::createDebugEvent(const bool autoReset) { - // create the event object - osEventHandle shaderEvent = osEventCreate(!autoReset); - - // event object has been created, set the initial state - if (shaderEvent != 0) { - osEventReset(shaderEvent); // initial state is non-signaled - - if (device()->gslCtx()->exceptionNotification(shaderEvent)) { - return shaderEvent; - } - } - - return 0; -} - -int32_t GpuDebugManager::waitDebugEvent(DebugEvent pEvent, uint32_t timeOut) const { - if (osEventTimedWait(pEvent, timeOut)) { - return CL_SUCCESS; - } else { - return CL_EVENT_TIMEOUT_AMD; - } -} - -void GpuDebugManager::destroyDebugEvent(DebugEvent* pEvent) { - osEventDestroy(*pEvent); - *pEvent = 0; - - device()->gslCtx()->exceptionNotification(0); -} - -void GpuDebugManager::wavefrontControl(uint32_t waveAction, uint32_t waveMode, uint32_t trapId, - void* waveAddr) const { - device()->gslCtx()->executeSqCommand(waveAction, waveMode, trapId, waveAddr); -} - -void GpuDebugManager::setAddressWatch(uint32_t numWatchPoints, void** watchAddress, - uint64_t* watchMask, uint64_t* watchMode, DebugEvent* pEvent) { - size_t requiredSize = numWatchPoints * sizeof(HwDbgAddressWatch); - - // previously allocated size is not big enough, allocate new memory - if (addressWatchSize_ < requiredSize) { - if (NULL != addressWatch_) { // free the smaller address watch storage - delete[] addressWatch_; - } - addressWatch_ = new HwDbgAddressWatch[numWatchPoints]; - addressWatchSize_ = requiredSize; - } - - // fill in the address watch structure - memset(addressWatch_, 0, addressWatchSize_); - - for (uint32_t i = 0; i < numWatchPoints; i++) { - amd::Memory* watchMem = as_amd(reinterpret_cast(watchAddress[i])); - Memory* watchMemAddress = device()->getGpuMemory(watchMem); - - addressWatch_[i].watchAddress_ = reinterpret_cast(watchMemAddress->vmAddress()); - addressWatch_[i].watchMask_ = watchMask[i]; - addressWatch_[i].watchMode_ = (cl_dbg_address_watch_mode_amd)watchMode[i]; - addressWatch_[i].event_ = (0 != event) ? event[i] : 0; - } - - // setup the watch addresses - device()->gslCtx()->setAddressWatch(numWatchPoints, (void*)addressWatch_); -} - -void GpuDebugManager::setGlobalMemory(amd::Memory* memObj, uint32_t offset, void* srcPtr, - uint32_t size) { - gpu::Memory* globalMem = device()->getGpuMemory(memObj); - - address mappedMem = static_cast
(globalMem->map(NULL, 0)); - assert(mappedMem != 0); - - void* dest_ptr = reinterpret_cast(mappedMem + offset); - memcpy(dest_ptr, srcPtr, size); - - globalMem->unmap(NULL); -} - -int32_t GpuDebugManager::createRuntimeTrapHandler() { - size_t codeSize = 0; - const uint32_t* rtTrapCode = NULL; - - if (device()->settings().viPlus_) { - codeSize = sizeof(RuntimeTrapCodeVi); - rtTrapCode = RuntimeTrapCodeVi; - } else { - codeSize = sizeof(RuntimeTrapCode); - rtTrapCode = RuntimeTrapCode; - } - - uint32_t numCodes = codeSize / sizeof(uint32_t); - - // Handle TMA corruption hw bug workaround - - // The trap handler buffer has extra 256 bytes allocated, the TMA address - // is stored in the first two DWORDs and the actual trap handler code - // is stored starting at the location of 256 bytes (TbaStartOffset). - // - // allocate memory for the runtime trap handler (TBA) + TMA address - uint32_t allocSize = codeSize + TbaStartOffset; - - Memory* rtTBA = new Memory(*device(), allocSize); - runtimeTBA_ = rtTBA; - - if ((rtTBA == NULL) || !rtTBA->create(Resource::RemoteUSWC)) { - return CL_OUT_OF_RESOURCES; - } - address tbaAddress = reinterpret_cast
(rtTBA->map(NULL)); - - // allocate buffer for the runtime trap handler buffer (TMA) - uint32_t tmaSize = 0x100; - Memory* rtTMA = new Memory(*device(), tmaSize); - runtimeTMA_ = rtTMA; - - if ((rtTMA == NULL) || !rtTMA->create(Resource::RemoteUSWC)) { - return CL_OUT_OF_RESOURCES; - } - - uint64_t rtTmaAddress = rtTMA->vmAddress(); - if ((rtTBA->vmAddress() & 0xFF) != 0 || (rtTmaAddress & 0xFF) != 0) { - LogError("debugmanager: Trap handler/buffer is not 256-byte aligned"); - return CL_INVALID_VALUE; - } - - // store the TMA address at the beginning of trap handler buffer - uint64_t* tbaStorage = reinterpret_cast(tbaAddress); - tbaStorage[0] = rtTmaAddress; - - // save the trap handler code - uint32_t* trapHandlerPtr = (uint32_t*)(tbaAddress + TbaStartOffset); - for (uint32_t i = 0; i < numCodes; i++) { - trapHandlerPtr[i] = rtTrapCode[i]; - } - - rtTBA->unmap(NULL); - - return CL_SUCCESS; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpudebugmanager.hpp b/projects/clr/rocclr/device/gpu/gpudebugmanager.hpp deleted file mode 100644 index 18e6ad336c..0000000000 --- a/projects/clr/rocclr/device/gpu/gpudebugmanager.hpp +++ /dev/null @@ -1,130 +0,0 @@ -/* Copyright (c) 2014 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef HWDBG_DEBUGMANAGER_H__ -#define HWDBG_DEBUGMANAGER_H__ - -#include "gpuvirtual.hpp" -#include "gpudebugger.hpp" - -namespace gpu { - -class GpuDebugManager; -class Device; -class Memory; - - -/*! \brief Debug Manager Class - * - * The debug manager class is used to pass all the trap info to the - * kernel dispatch and then the kernel execution can use such trap information - * for kernel execution. This class contains the trap handler and shader event - * objects. The trap handler is setup by users and passed to the kernel dispatch. - * The shader event is to receive interrupts from the GPU and then users can - * perform various operations. - * - * This class also provides the interface for setting up the pre-dispatch - * callback functions used by the profiler and debugger. It also provides - * a way to retrieve various debug information for the kernel execution. - * - */ -class GpuDebugManager : public amd::HwDebugManager { - public: - //! Constructor of the debug manager class - GpuDebugManager(amd::Device* device); - - //! Destructor of the debug manager class - ~GpuDebugManager(); - - //! Get the single instance of the GpuDebugManager class - static GpuDebugManager* getDefaultInstance(); - - //! Destroy the GpuDebugManager class object - static void destroyInstances(); - - //! Flush cache - void flushCache(uint32_t mask); - - //! Create the debug event - DebugEvent createDebugEvent(const bool autoReset); - - //! Wait for the debug event - int32_t waitDebugEvent(DebugEvent pEvent, uint32_t timeOut) const; - - //! Destroy the debug event - void destroyDebugEvent(DebugEvent* pEvent); - - //! Register the debugger - int32_t registerDebugger(amd::Context* context, uintptr_t messageStorage); - - //! Unregister the debugger - void unregisterDebugger(); - - //! Send the wavefront control cmmand - void wavefrontControl(uint32_t waveAction, uint32_t waveMode, uint32_t trapId, - void* waveAddr) const; - - //! Set address watching point - void setAddressWatch(uint32_t numWatchPoints, void** watchAddress, uint64_t* watchMask, - uint64_t* watchMode, DebugEvent* pEvent); - - //! Map the kernel code for host access - void mapKernelCode(void* aqlCodeInfo) const; - - //! Get the packet information for dispatch - void getPacketAmdInfo(const void* aqlCodeInfo, void* packetInfo) const; - - //! Set global memory values - void setGlobalMemory(amd::Memory* memObj, uint32_t offset, void* srcPtr, uint32_t size); - - //! Execute the post-dispatch callback function - void executePostDispatchCallBack(); - - //! Execute the pre-dispatch callback function - void executePreDispatchCallBack(void* aqlPacket, void* toolInfo); - - private: - //! Setup trap handler info for kernel execution - void setupTrapInformation(DebugToolInfo* toolInfo); - - //! Create runtime trap handler - int32_t createRuntimeTrapHandler(); - - protected: - const VirtualGPU* vGpu() const { return vGpu_; } - - private: - const gpu::Device* device() const { return reinterpret_cast(device_); } - - VirtualGPU* vGpu_; //!< the virtual GPU - - uintptr_t debugMessages_; //!< Pointer to a SHARED_DEBUG_MESSAGES pass to the KMD - - HwDbgAddressWatch* addressWatch_; //!< Address watch data - size_t addressWatchSize_; //!< Size of address watch data - - //! Arguments used by the callback function - void* oclEventHandle_; //!< event handler - const hsa_kernel_dispatch_packet_t* aqlPacket_; //!< AQL packet -}; - -} // namespace gpu - -#endif // HWDBG_DEBUGMANAGER_H__ diff --git a/projects/clr/rocclr/device/gpu/gpudefs.hpp b/projects/clr/rocclr/device/gpu/gpudefs.hpp deleted file mode 100644 index 6bf8f4a976..0000000000 --- a/projects/clr/rocclr/device/gpu/gpudefs.hpp +++ /dev/null @@ -1,354 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUDEFS_HPP_ -#define GPUDEFS_HPP_ - -#include "top.hpp" - -#include "gsl_types.h" -#include "gsl_config.h" -#include "gsl_ctx.h" -#include "backend.h" -#include "GSLDevice.h" -#include "GSLContext.h" - -extern bool getFuncInfoFromImage(CALimage image, CALfuncInfo* pFuncInfo); - -/*! \addtogroup GPU - * @{ - */ - -//! GPU Device Implementation - -namespace gpu { - -//! Maximum number of the supported global atomic counters -static constexpr uint MaxAtomicCounters = 8; -//! Maximum number of the supported samplers -static constexpr uint MaxSamplers = 16; -//! Maximum number of supported read images -static constexpr uint MaxReadImage = 128; -//! Maximum number of supported write images -static constexpr uint MaxWriteImage = 8; -//! Maximum number of supported read/write images for OCL20 -static constexpr uint MaxReadWriteImage = 64; -//! Maximum number of supported constant arguments -static constexpr uint MaxConstArguments = 8; -//! Maximum number of supported kernel UAV arguments -static constexpr uint MaxUavArguments = 1024; -//! Maximum number of pixels for a 1D image created from a buffer -static constexpr size_t MaxImageBufferSize = 1 << 27; -//! Maximum number of pixels for a 1D image created from a buffer -static constexpr size_t MaxImageArraySize = 2048; - -//! Maximum number of supported constant buffers -static constexpr uint MaxConstBuffers = MaxConstArguments + 8; - -//! Maximum number of constant buffers for arguments -static constexpr uint MaxConstBuffersArguments = 2; - -//! Define offline CAL implementation -static constexpr uint CalOfflineImpl = 0xffffffff; - -//! Alignment restriciton for the pinned memory -static constexpr size_t PinnedMemoryAlignment = 4 * Ki; - -//! HSA path specific defines for images -static constexpr uint HsaImageObjectSize = 48; -static constexpr uint HsaImageObjectAlignment = 16; -static constexpr uint HsaSamplerObjectSize = 32; -static constexpr uint HsaSamplerObjectAlignment = 16; - -//! HSA path specific defines for images -static constexpr uint DeviceQueueMaskSize = 32; - -// Supported OpenCL versions -enum OclVersion { OpenCL10, OpenCL11, OpenCL12, OpenCL20, OpenCL21 }; - -struct CalFormat { - gslChannelOrder channelOrder_; //!< Texel/pixel GSL channel order - cmSurfFmt type_; //!< Texel/pixel CAL format -}; - -struct MemoryFormat { - cl_image_format clFormat_; //!< CL image format - CalFormat calFormat_; //!< CAL image format -}; - -static constexpr MemoryFormat MemoryFormatMap[] = { - // R - {{CL_R, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_INTENSITY8}}, - {{CL_R, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_R16}}, - - {{CL_R, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_sR8}}, - {{CL_R, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_sU16}}, - - {{CL_R, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_sR8I}}, - {{CL_R, CL_SIGNED_INT16}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_sR16I}}, - {{CL_R, CL_SIGNED_INT32}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_sR32I}}, - {{CL_R, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_R8I}}, - {{CL_R, CL_UNSIGNED_INT16}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_R16I}}, - {{CL_R, CL_UNSIGNED_INT32}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_R32I}}, - - {{CL_R, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_R16F}}, - {{CL_R, CL_FLOAT}, {GSL_CHANNEL_ORDER_R, CM_SURF_FMT_R32F}}, - - // A - {{CL_A, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_INTENSITY8}}, - {{CL_A, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_R16}}, - - {{CL_A, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_sR8}}, - {{CL_A, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_sU16}}, - - {{CL_A, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_sR8I}}, - {{CL_A, CL_SIGNED_INT16}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_sR16I}}, - {{CL_A, CL_SIGNED_INT32}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_sR32I}}, - {{CL_A, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_R8I}}, - {{CL_A, CL_UNSIGNED_INT16}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_R16I}}, - {{CL_A, CL_UNSIGNED_INT32}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_R32I}}, - - {{CL_A, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_R16F}}, - {{CL_A, CL_FLOAT}, {GSL_CHANNEL_ORDER_A, CM_SURF_FMT_R32F}}, - - // RG - {{CL_RG, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG8}}, - {{CL_RG, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG16}}, - - {{CL_RG, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_sRG8}}, - {{CL_RG, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_sUV16}}, - - {{CL_RG, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_sRG8I}}, - {{CL_RG, CL_SIGNED_INT16}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_sRG16I}}, - {{CL_RG, CL_SIGNED_INT32}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_sRG32I}}, - {{CL_RG, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG8I}}, - {{CL_RG, CL_UNSIGNED_INT16}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG16I}}, - {{CL_RG, CL_UNSIGNED_INT32}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG32I}}, - - {{CL_RG, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG16F}}, - {{CL_RG, CL_FLOAT}, {GSL_CHANNEL_ORDER_RG, CM_SURF_FMT_RG32F}}, - - // RA - {{CL_RA, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG8}}, - {{CL_RA, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG16}}, - - {{CL_RA, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG8}}, - {{CL_RA, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sUV16}}, - - {{CL_RA, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG8I}}, - {{CL_RA, CL_SIGNED_INT16}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG16I}}, - {{CL_RA, CL_SIGNED_INT32}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG32I}}, - {{CL_RA, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG8I}}, - {{CL_RA, CL_UNSIGNED_INT16}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG16I}}, - {{CL_RA, CL_UNSIGNED_INT32}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG32I}}, - - {{CL_RA, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG16F}}, - {{CL_RA, CL_FLOAT}, {GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG32F}}, - - // RGB - {{CL_RGB, CL_UNORM_INT_101010}, {GSL_CHANNEL_ORDER_RGB, CM_SURF_FMT_BGR10_X2}}, - // RGBA - {{CL_RGBA, CL_UNORM_INT_101010}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGB10_X2}}, - - // RGBA - {{CL_RGBA, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA8}}, - {{CL_RGBA, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA16}}, - - {{CL_RGBA, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_sRGBA8}}, - {{CL_RGBA, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_sUVWQ16}}, - - {{CL_RGBA, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_sRGBA8I}}, - {{CL_RGBA, CL_SIGNED_INT16}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_sRGBA16I}}, - {{CL_RGBA, CL_SIGNED_INT32}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_sRGBA32I}}, - {{CL_RGBA, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA8UI}}, - {{CL_RGBA, CL_UNSIGNED_INT16}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA16UI}}, - {{CL_RGBA, CL_UNSIGNED_INT32}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA32UI}}, - - {{CL_RGBA, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA16F}}, - {{CL_RGBA, CL_FLOAT}, {GSL_CHANNEL_ORDER_RGBA, CM_SURF_FMT_RGBA32F}}, - - // ARGB - {{CL_ARGB, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_ARGB, CM_SURF_FMT_RGBA8}}, - {{CL_ARGB, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_ARGB, CM_SURF_FMT_sRGBA8}}, - {{CL_ARGB, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_ARGB, CM_SURF_FMT_sRGBA8I}}, - {{CL_ARGB, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_ARGB, CM_SURF_FMT_RGBA8UI}}, - - // BGRA - {{CL_BGRA, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_BGRA, CM_SURF_FMT_RGBA8}}, - {{CL_BGRA, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_BGRA, CM_SURF_FMT_sRGBA8}}, - {{CL_BGRA, CL_SIGNED_INT8}, {GSL_CHANNEL_ORDER_BGRA, CM_SURF_FMT_sRGBA8I}}, - {{CL_BGRA, CL_UNSIGNED_INT8}, {GSL_CHANNEL_ORDER_BGRA, CM_SURF_FMT_RGBA8UI}}, - - // LUMINANCE - {{CL_LUMINANCE, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_LUMINANCE, CM_SURF_FMT_sR8}}, - {{CL_LUMINANCE, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_LUMINANCE, CM_SURF_FMT_sU16}}, - {{CL_LUMINANCE, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_LUMINANCE, CM_SURF_FMT_INTENSITY8}}, - {{CL_LUMINANCE, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_LUMINANCE, CM_SURF_FMT_R16}}, - {{CL_LUMINANCE, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_LUMINANCE, CM_SURF_FMT_R16F}}, - {{CL_LUMINANCE, CL_FLOAT}, {GSL_CHANNEL_ORDER_LUMINANCE, CM_SURF_FMT_R32F}}, - - // INTENSITY - {{CL_INTENSITY, CL_SNORM_INT8}, {GSL_CHANNEL_ORDER_INTENSITY, CM_SURF_FMT_sR8}}, - {{CL_INTENSITY, CL_SNORM_INT16}, {GSL_CHANNEL_ORDER_INTENSITY, CM_SURF_FMT_sU16}}, - {{CL_INTENSITY, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_INTENSITY, CM_SURF_FMT_INTENSITY8}}, - {{CL_INTENSITY, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_INTENSITY, CM_SURF_FMT_R16}}, - {{CL_INTENSITY, CL_HALF_FLOAT}, {GSL_CHANNEL_ORDER_INTENSITY, CM_SURF_FMT_R16F}}, - {{CL_INTENSITY, CL_FLOAT}, {GSL_CHANNEL_ORDER_INTENSITY, CM_SURF_FMT_R32F}}, - - // sRBGA - {{CL_sRGBA, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_SRGBA, CM_SURF_FMT_RGBA8_SRGB}}, - {{CL_sRGBA, CL_UNSIGNED_INT8}, // This is used only by blit kernel - {GSL_CHANNEL_ORDER_SRGBA, CM_SURF_FMT_RGBA8UI}}, - - // sRBG - {{CL_sRGB, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_SRGB, CM_SURF_FMT_RGBX8UI}}, - {{CL_sRGB, CL_UNSIGNED_INT8}, // This is used only by blit kernel - {GSL_CHANNEL_ORDER_SRGB, CM_SURF_FMT_RGBA8UI}}, - - // sRBGx - {{CL_sRGBx, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_SRGBX, CM_SURF_FMT_RGBX8UI}}, - {{CL_sRGBx, CL_UNSIGNED_INT8}, // This is used only by blit kernel - {GSL_CHANNEL_ORDER_SRGBX, CM_SURF_FMT_RGBA8UI}}, - - // sBGRA - {{CL_sBGRA, CL_UNORM_INT8}, {GSL_CHANNEL_ORDER_SBGRA, CM_SURF_FMT_RGBA8}}, - {{CL_sBGRA, CL_UNSIGNED_INT8}, // This is used only by blit kernel - {GSL_CHANNEL_ORDER_SBGRA, CM_SURF_FMT_RGBA8UI}}, - - // DEPTH - {{CL_DEPTH, CL_FLOAT}, {GSL_CHANNEL_ORDER_REPLICATE_R, CM_SURF_FMT_DEPTH32F}}, - {{CL_DEPTH, CL_UNSIGNED_INT32}, // This is used only by blit kernel - {GSL_CHANNEL_ORDER_REPLICATE_R, CM_SURF_FMT_R32I}}, - - {{CL_DEPTH, CL_UNORM_INT16}, {GSL_CHANNEL_ORDER_REPLICATE_R, CM_SURF_FMT_DEPTH16}}, - {{CL_DEPTH, CL_UNSIGNED_INT16}, // This is used only by blit kernel - {GSL_CHANNEL_ORDER_REPLICATE_R, CM_SURF_FMT_R16I}}, - - {{CL_DEPTH_STENCIL, CL_UNORM_INT24}, - {GSL_CHANNEL_ORDER_REPLICATE_R, CM_SURF_FMT_DEPTH24_STEN8}}, - {{CL_DEPTH_STENCIL, CL_FLOAT}, {GSL_CHANNEL_ORDER_REPLICATE_R, CM_SURF_FMT_DEPTH32F_X24_STEN8}} - -}; - -struct MemFormatStruct { - cmSurfFmt format_; - uint size_; - uint components_; -}; - -static constexpr MemFormatStruct MemoryFormatSize[] = { - {CM_SURF_FMT_INTENSITY8, 1, - 1}, /**< 1 component, normalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_RG8, 2, - 2}, /**< 2 component, normalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_RGBA8, 4, - 4}, /**< 4 component, normalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_RGBA8_SRGB, 4, - 4}, /**< 4 component, normalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_R16, 2, - 1}, /**< 1 component, normalized unsigned 16-bit integer value per component */ - {CM_SURF_FMT_RG16, 4, - 2}, /**< 2 component, normalized unsigned 16-bit integer value per component */ - {CM_SURF_FMT_RGBA16, 8, - 4}, /**< 4 component, normalized unsigned 16-bit integer value per component */ - {CM_SURF_FMT_sRGBA8, 4, - 4}, /**< 4 component, normalized signed 8-bit integer value per component */ - {CM_SURF_FMT_sU16, 2, - 1}, /**< 1 component, normalized signed 16-bit integer value per component */ - {CM_SURF_FMT_sUV16, 4, - 2}, /**< 2 component, normalized signed 16-bit integer value per component */ - {CM_SURF_FMT_sUVWQ16, 8, - 4}, /**< 4 component, normalized signed 16-bit integer value per component */ - {CM_SURF_FMT_R32F, 4, 1}, /**< A 1 component, 32-bit float value per component */ - {CM_SURF_FMT_RG32F, 8, 2}, /**< A 2 component, 32-bit float value per component */ - {CM_SURF_FMT_RGBA32F, 16, 4}, /**< A 4 component, 32-bit float value per component */ - {CM_SURF_FMT_sR8, 1, - 1}, /**< 1 component, normalized signed 8-bit integer value per component */ - {CM_SURF_FMT_sRG8, 2, - 2}, /**< 2 component, normalized signed 8-bit integer value per component */ - - {CM_SURF_FMT_R8I, 1, - 1}, /**< 1 component, unnormalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_RG8I, 2, - 2}, /**< 2 component, unnormalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_RGBA8UI, 4, - 4}, /**< 4 component, unnormalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_RGBX8UI, 4, - 4}, /**< 4 component, unnormalized unsigned 8-bit integer value per component */ - {CM_SURF_FMT_sR8I, 1, - 1}, /**< 1 component, unnormalized signed 8-bit integer value per component */ - {CM_SURF_FMT_sRG8I, 2, - 2}, /**< 2 component, unnormalized signed 8-bit integer value per component */ - {CM_SURF_FMT_sRGBA8I, 4, - 4}, /**< 4 component, unnormalized signed 8-bit integer value per component */ - {CM_SURF_FMT_R16I, 2, - 1}, /**< 1 component, unnormalized unsigned 16-bit integer value per component */ - {CM_SURF_FMT_RG16I, 4, - 2}, /**< 2 component, unnormalized unsigned 16-bit integer value per component */ - {CM_SURF_FMT_RGBA16UI, 8, - 4}, /**< 4 component, unnormalized unsigned 16-bit integer value per component */ - {CM_SURF_FMT_sR16I, 2, - 1}, /**< 1 component, unnormalized signed 16-bit integer value per component */ - {CM_SURF_FMT_sRG16I, 4, - 2}, /**< 2 component, unnormalized signed 16-bit integer value per component */ - {CM_SURF_FMT_sRGBA16I, 8, - 4}, /**< 4 component, unnormalized signed 16-bit integer value per component */ - {CM_SURF_FMT_R32I, 4, - 1}, /**< 1 component, unnormalized unsigned 32-bit integer value per component */ - {CM_SURF_FMT_RG32I, 8, - 2}, /**< 2 component, unnormalized unsigned 32-bit integer value per component */ - {CM_SURF_FMT_RGBA32UI, 16, - 4}, /**< 4 component, unnormalized unsigned 32-bit integer value per component */ - {CM_SURF_FMT_sR32I, 4, - 1}, /**< 1 component, unnormalized signed 32-bit integer value per component */ - {CM_SURF_FMT_sRG32I, 8, - 2}, /**< 2 component, unnormalized signed 32-bit integer value per component */ - {CM_SURF_FMT_sRGBA32I, 16, - 4}, /**< 4 component, unnormalized signed 32-bit integer value per component */ - - {CM_SURF_FMT_R16F, 2, 1}, /**< A 1 component, 16-bit float value per component */ - {CM_SURF_FMT_RG16F, 4, 2}, /**< A 2 component, 16-bit float value per component */ - {CM_SURF_FMT_RGBA16F, 8, 4}, /**< A 4 component, 16-bit float value per component */ - - {CM_SURF_FMT_BGR10_X2, 4, 4}, /**< 4 component, unnormalized signed 10-bit integer value per - component packed as (@c XXRRRRRRRRRRGGGGGGGGGGBBBBBBBBBB)*/ - {CM_SURF_FMT_RGB10_X2, 4, 4}, /**< 4 component, unnormalized signed 10-bit integer value per - component packed as (@c XXRRRRRRRRRRGGGGGGGGGGBBBBBBBBBB)*/ - {CM_SURF_FMT_DEPTH32F, 4, 1}, /**< A one component, 32 float value per component */ - {CM_SURF_FMT_DEPTH16, 2, 1}, /**< A one component, 16 unsigned int value per component */ - {CM_SURF_FMT_DEPTH24_STEN8, 4, 1}, /**< A one component, 32 float value per component */ - {CM_SURF_FMT_DEPTH32F_X24_STEN8, 8, - 2} /**< depth + stencil, 64 bits per element packed as (@c - XXXXXXXXXXXXXXXXXXXXXXXXSSSSSSSSDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD) */ -}; - -__inline const MemFormatStruct& memoryFormatSize(cmSurfFmt fmt) { - for (uint i = 0; i < sizeof(MemoryFormatSize) / sizeof(MemFormatStruct); ++i) { - if (MemoryFormatSize[i].format_ == fmt) { - return MemoryFormatSize[i]; - } - } - assert(!"Unknown GSL memory format!"); - return MemoryFormatSize[0]; -} - -} // namespace gpu - -#endif // GPUDEFS_HPP_ diff --git a/projects/clr/rocclr/device/gpu/gpudevice.cpp b/projects/clr/rocclr/device/gpu/gpudevice.cpp deleted file mode 100644 index 6aab22a46a..0000000000 --- a/projects/clr/rocclr/device/gpu/gpudevice.cpp +++ /dev/null @@ -1,2263 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "platform/program.hpp" -#include "platform/kernel.hpp" -#include "os/os.hpp" -#include "device/device.hpp" -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpumemory.hpp" -#include "device/gpu/gpudevice.hpp" -#include "utils/flags.hpp" -#include "utils/versions.hpp" -#include "utils/options.hpp" -#include "thread/monitor.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpubinary.hpp" -#include "device/gpu/gpusettings.hpp" -#include "device/gpu/gpublit.hpp" -#include "cz_id.h" - -#include "hsailctx.hpp" - -#include "vdi_common.hpp" -#include "CL/cl_gl.h" - -#ifdef _WIN32 -#include -#include -#include "CL/cl_d3d10.h" -#include "CL/cl_d3d11.h" -#include "CL/cl_dx9_media_sharing.h" -#endif // _WIN32 - -#include "os_if.h" // for osInit() -#include "gpudebugmanager.hpp" - -#include -#include -#include -#include -#include -#include - -namespace { - -//! Define the mapping from CAL asic enumeration values to the -//! compiler gfx major/minor/stepping version. -struct CalDevice { - uint32_t gfxipMajor_; //!< The core engine GFXIP Major version - uint32_t gfxipMinor_; //!< The core engine GFXIP Minor version - uint32_t gfxipStepping_; //!< The core engine GFXIP Stepping version - CALMachineType calMachine_; //!< CAL machine type - const char* calName_; //!< CAL device name - CALtarget calTarget_; //!< CAL target - bool preferPal_; //!< Prefer to use PAL if GPU_ENABLE_PAL=2 - bool nullUseDouble_; //!< Use double precision for a NullDevice - bool nullUseOpenCL200_; //!< Use OpenCL 2.0 for a NullDevice -}; - -static constexpr CalDevice supportedCalDevices[] = { -// Prefer - NullDevice - -// GFX Version GSL Machine CAL Name CAL Target PAL double OCL200 - {7, 0, 0, ED_ATI_CAL_MACHINE_KALINDI_ISA, "Kalindi", CAL_TARGET_KALINDI, false, true, true }, - {7, 0, 0, ED_ATI_CAL_MACHINE_SPECTRE_ISA, "Spectre", CAL_TARGET_SPECTRE, false, true, true }, - {7, 0, 0, ED_ATI_CAL_MACHINE_SPOOKY_ISA, "Spooky", CAL_TARGET_SPOOKY, false, true, true }, - {7, 0, 2, ED_ATI_CAL_MACHINE_HAWAII_ISA, "Hawaii", CAL_TARGET_HAWAII, false, true, true }, // Also Hawaiipro (generated code is for Hawaiipro) - {7, 0, 4, ED_ATI_CAL_MACHINE_BONAIRE_ISA, "Bonaire", CAL_TARGET_BONAIRE, false, true, true }, - {7, 0, 5, ED_ATI_CAL_MACHINE_GODAVARI_ISA, "Mullins", CAL_TARGET_GODAVARI, false, true, true }, // FIXME: Why is this compiled as Mullins yet reported as Godavari? Add gfx703 to support Mullins. - {8, 0, 1, ED_ATI_CAL_MACHINE_CARRIZO_ISA, "Carrizo", CAL_TARGET_CARRIZO, false, true, true }, // Also Bristol Ridge - {8, 0, 2, ED_ATI_CAL_MACHINE_ICELAND_ISA, "Iceland", CAL_TARGET_ICELAND, false, true, true }, - {8, 0, 2, ED_ATI_CAL_MACHINE_TONGA_ISA, "Tonga", CAL_TARGET_TONGA, false, true, true }, // Also Tongapro (generated code is for Tonga) - {8, 0, 3, ED_ATI_CAL_MACHINE_FIJI_ISA, "Fiji", CAL_TARGET_FIJI, false, true, true }, - {8, 0, 3, ED_ATI_CAL_MACHINE_ELLESMERE_ISA, "Ellesmere", CAL_TARGET_ELLESMERE, false, true, true }, // Polaris10 - {8, 0, 3, ED_ATI_CAL_MACHINE_BAFFIN_ISA, "Baffin", CAL_TARGET_BAFFIN, false, true, true }, // Polaris11 - {8, 0, 3, ED_ATI_CAL_MACHINE_LEXA_ISA, "gfx803", CAL_TARGET_LEXA, false, true, true }, // Polaris12 -#if !defined(BRAHMA) - {8, 0, 3, ED_ATI_CAL_MACHINE_POLARIS22_ISA, "gfx803", CAL_TARGET_POLARIS22, false, true, true }, -#endif - {8, 1, 0, ED_ATI_CAL_MACHINE_STONEY_ISA, "Stoney", CAL_TARGET_STONEY, false, true, true }, -#if !defined(BRAHMA) - {9, 0, 0, ED_ATI_CAL_MACHINE_GREENLAND_ISA, "gfx900", CAL_TARGET_GREENLAND, true, true, true }, // Vega10 - {9, 0, 2, ED_ATI_CAL_MACHINE_RAVEN_ISA, "gfx902", CAL_TARGET_RAVEN, true, true, true }, - {9, 0, 4, ED_ATI_CAL_MACHINE_VEGA12_ISA, "gfx904", CAL_TARGET_VEGA12, true, true, true }, - {9, 0, 6, ED_ATI_CAL_MACHINE_VEGA20_ISA, "gfx906", CAL_TARGET_VEGA20, true, true, true }, - {9, 0, 9, ED_ATI_CAL_MACHINE_RAVEN2_ISA, "gfx909", CAL_TARGET_RAVEN2, true, true, true }, - {9, 0, 12, ED_ATI_CAL_MACHINE_RENOIR_ISA, "gfx90c", CAL_TARGET_RENOIR, true, true, true }, -#endif -}; -static_assert(CAL_TARGET_LAST == CAL_TARGET_VEGA20, "Add new CAL targets to mapping"); - -static std::tuple findIsa( - CALtarget calTarget, bool sramecc, bool xnack) { - auto calDeviceIter = - std::find_if(std::begin(supportedCalDevices), std::end(supportedCalDevices), - [&](const CalDevice& calDevice) { return calDevice.calTarget_ == calTarget; }); - if (calDeviceIter == std::end(supportedCalDevices)) { - return std::make_tuple(nullptr, static_cast(0), nullptr, false, false, false); - } - const amd::Isa* isa = amd::Isa::findIsa( - calDeviceIter->gfxipMajor_, calDeviceIter->gfxipMinor_, calDeviceIter->gfxipStepping_, - sramecc ? amd::Isa::Feature::Enabled : amd::Isa::Feature::Disabled, - xnack ? amd::Isa::Feature::Enabled : amd::Isa::Feature::Disabled); - return std::make_tuple(isa, calDeviceIter->calMachine_, calDeviceIter->calName_, - calDeviceIter->preferPal_, calDeviceIter->nullUseDouble_, - calDeviceIter->nullUseOpenCL200_); -} - -static std::tuple findCal( - uint32_t gfxipMajor, uint32_t gfxipMinor, uint32_t gfxipStepping) { - auto calDeviceIter = std::find_if(std::begin(supportedCalDevices), std::end(supportedCalDevices), - [&](const CalDevice& calDevice) { - return calDevice.gfxipMajor_ == gfxipMajor && - calDevice.gfxipMinor_ == gfxipMinor && - calDevice.gfxipStepping_ == gfxipStepping; - }); - if (calDeviceIter == std::end(supportedCalDevices)) { - return std::make_tuple(false, static_cast(0), static_cast(0), - nullptr, false, false, false); - } - return std::make_tuple(true, calDeviceIter->calMachine_, calDeviceIter->calTarget_, - calDeviceIter->calName_, calDeviceIter->preferPal_, - calDeviceIter->nullUseDouble_, calDeviceIter->nullUseOpenCL200_); -} - -} // namespace - -bool DeviceLoad() { - bool ret = false; - - // Create online devices - ret |= gpu::Device::init(); - // Create offline GPU devices - ret |= gpu::NullDevice::init(); - - return ret; -} - -void DeviceUnload() { gpu::Device::tearDown(); } - -namespace gpu { - -aclCompiler* NullDevice::compiler_; -aclCompiler* NullDevice::hsaCompiler_; -AppProfile Device::appProfile_; - -NullDevice::NullDevice() - : amd::Device(), - calTarget_(static_cast(0)), - calMachine_(static_cast(0)), - calName_(nullptr) {} - -bool NullDevice::init() { - // Create offline devices for all ISAs not already associated with an online - // device. This allows code objects to be compiled for all supported ISAs. - std::vector devices = getDevices(CL_DEVICE_TYPE_GPU, false); - for (const amd::Isa *isa = amd::Isa::begin(); isa != amd::Isa::end(); isa++) { - if (!isa->runtimeGslSupported()) { - continue; - } - bool isOnline = false; - // Check if the particular device is online - for (size_t i = 0; i < devices.size(); i++) { - if (&(devices[i]->isa()) == isa) { - isOnline = true; - break; - } - } - if (isOnline) { - continue; - } - - bool found; - CALMachineType calMachine; - CALtarget calTarget; - const char* calName; - bool preferPal; - bool nullUseDouble; - bool nullUseOpenCL200; - std::tie(found, calMachine, calTarget, calName, preferPal, nullUseDouble, nullUseOpenCL200) = - findCal(isa->versionMajor(), isa->versionMinor(), isa->versionStepping()); - if (!found) { - // GSL does not support this asic. - continue; - } - - std::unique_ptr nullDevice(new NullDevice()); - if (!nullDevice) { - LogPrintfError("Error allocating new instance of offline CAL Device %s", isa->targetId()); - return false; - } - if (!nullDevice->create(calName, *isa, calTarget, preferPal, nullUseDouble, nullUseOpenCL200)) { - // Skip over unsupported devices - LogPrintfError("Skipping creating new instance of offline CAL Device %s", isa->targetId()); - continue; - } - nullDevice.release()->registerDevice(); - } - return true; -} - -bool NullDevice::create(const char* calName, const amd::Isa& isa, CALtarget target, - bool preferPal, bool doublePrecision, bool openCL200) { - if (!isa.runtimeGslSupported()) { - LogPrintfError("Offline CAL device %s is not supported", isa.targetId()); - return false; - } - if ((GPU_ENABLE_PAL == 2) && isa.runtimePalSupported() && preferPal) { - LogPrintfError("Skipping as GPU_ENABLE_PAL=2 indicating to use PAL for offline CAL device %s", - isa.targetId()); - return false; - } - - online_ = false; - calTarget_ = target; - calName_ = calName; - - // sets up vaCacheAccess_ and vaCacheMap_. - if (!amd::Device::create(isa)) { - LogPrintfError("Unable to setup offline device for CAL device %s", isa.targetId()); - return false; - } - - CALdeviceattribs calAttr = {0}; - calAttr.target = calTarget(); - // Force double if it could be supported - if (doublePrecision) { - calAttr.doublePrecision = CAL_TRUE; - } - // Use OpenCL 2.0 if supported - if (openCL200) { - calAttr.isOpenCL200Device = CAL_TRUE; - } - - settings_ = new gpu::Settings(); - gpu::Settings* gpuSettings = reinterpret_cast(settings_); - // Create setting for the offline target - if ((gpuSettings == NULL) || !gpuSettings->create(calAttr)) { - LogPrintfError("GPU settings failed for offline device for CAL device %s", isa.targetId()); - return false; - } - - if (!ValidateHsail()) { - LogPrintfError("HSAIL initialization failed for offline CAL device %s", isa.targetId()); - return false; - } - - gslMemInfo memInfo = {0}; - // Report 512MB for all offline devices - memInfo.cardMemAvailableBytes = 512 * Mi; - memInfo.cardLargestFreeBlockBytes = 512 * Mi; - calAttr.localRAM = 512; - - // Fill the device info structure - fillDeviceInfo(calAttr, memInfo, 4096, 1, 0); - - // Runtime doesn't know what local size could be on the real board - info_.maxGlobalVariableSize_ = static_cast(512 * Mi); - - if (NULL == hsaCompiler_) { - const char* library = getenv("HSA_COMPILER_LIBRARY"); - aclCompilerOptions opts = { - sizeof(aclCompilerOptions_0_8), library, NULL, NULL, NULL, NULL, NULL, AMD_OCL_SC_LIB}; - // Initialize the compiler handle - acl_error error; - hsaCompiler_ = amd::Hsail::CompilerInit(&opts, &error); - if (error != ACL_SUCCESS) { - LogPrintfError("Error initializing the compiler for offline CAL device %s", isa.targetId()); - return false; - } - } - - return true; -} - -bool NullDevice::isHsailProgram(amd::option::Options* options) { - bool isCIPlus = settings().ciPlus_; - bool isBlit = false; - bool isSPIRV = false; - bool isClang = false; - bool isEDG = false; - bool isLegacy = false; - bool isOCL20 = false; - std::vector optvec; - bool isInputOptions = false; - if (options != NULL) { - optvec.push_back(options); - isInputOptions = true; - } - amd::option::Options parsedOptions; - constexpr bool OptionChangable = true; - constexpr bool LinkOptsOnly = false; - constexpr bool IsLC = false; - if (!amd::Program::ParseAllOptions("", parsedOptions, OptionChangable, LinkOptsOnly, IsLC)) { - return NULL; - } - optvec.push_back(&parsedOptions); - for (auto const op : optvec) { - // TODO: Remove isOCL20 related code from this function along with switching HSAIL by default - if (isCIPlus && amd::Program::GetOclCVersion(op->oVariables->CLStd) >= 20) { - isOCL20 = true; - } - if (op->oVariables->clInternalKernel) { - isBlit = true; - break; - } - if (!isLegacy) { - isLegacy = op->oVariables->Legacy; - } - // Checks Frontend option only from input *options, not from Env, - // because they might be only calculated by RT based on the binaries to link. - // -frontend is being queried now instead of -cl-std=CL2.0, because the last one - // is not an indicator for HSAIL path anymore. - // TODO: Revise these binary's target checks - // and possibly remove them after switching to HSAIL by default. - if (isInputOptions) { - if (!isClang) { - isClang = op->isCStrOptionsEqual(op->oVariables->Frontend, "clang"); - } - if (!isEDG) { - isEDG = op->isCStrOptionsEqual(op->oVariables->Frontend, "edg"); - } - } - if (!isSPIRV) { - isSPIRV = op->oVariables->BinaryIsSpirv; - } - isInputOptions = false; - } - if (isSPIRV || (isBlit && isCIPlus) || isClang || isOCL20) { - return true; - } - if (isLegacy || isEDG) { - return false; - } - return true; -} - -device::Program* NullDevice::createProgram(amd::Program& owner, amd::option::Options* options) { - if (isHsailProgram(options)) { - return new HSAILProgram(*this, owner); - } - return new NullProgram(*this, owner); -} - -void NullDevice::fillDeviceInfo(const CALdeviceattribs& calAttr, const gslMemInfo& memInfo, - size_t maxTextureSize, uint numComputeRings, - uint numComputeRingsRT) { - info_.type_ = CL_DEVICE_TYPE_GPU; - info_.vendorId_ = 0x1002; - info_.maxComputeUnits_ = calAttr.numberOfSIMD; - info_.maxBoostComputeUnits_ = calAttr.numberOfSIMD; - info_.maxWorkItemDimensions_ = 3; - info_.numberOfShaderEngines = calAttr.numberOfShaderEngines; - - // SI parts are scalar. Also, reads don't need to be 128-bits to get peak rates. - // For example, float4 is not faster than float as long as all threads fetch the same - // amount of data and the reads are coalesced. This is from the H/W team and confirmed - // through experimentation. May also be true on EG/NI, but no point in confusing - // developers now. - info_.nativeVectorWidthChar_ = info_.preferredVectorWidthChar_ = 4; - info_.nativeVectorWidthShort_ = info_.preferredVectorWidthShort_ = 2; - info_.nativeVectorWidthInt_ = info_.preferredVectorWidthInt_ = 1; - info_.nativeVectorWidthLong_ = info_.preferredVectorWidthLong_ = 1; - info_.nativeVectorWidthFloat_ = info_.preferredVectorWidthFloat_ = 1; - info_.nativeVectorWidthDouble_ = info_.preferredVectorWidthDouble_ = - (settings().checkExtension(ClKhrFp64)) ? 1 : 0; - info_.nativeVectorWidthHalf_ = info_.preferredVectorWidthHalf_ = 0; // no half support - - info_.maxEngineClockFrequency_ = (calAttr.engineClock != 0) ? calAttr.engineClock : 555; - info_.maxMemoryClockFrequency_ = (calAttr.memoryClock != 0) ? calAttr.memoryClock : 555; - info_.timeStampFrequency_ = 1000000; - info_.vramBusBitWidth_ = calAttr.memBusWidth; - info_.l2CacheSize_ = 0; - info_.maxParameterSize_ = 1024; - info_.minDataTypeAlignSize_ = sizeof(int64_t[16]); - info_.singleFPConfig_ = - CL_FP_ROUND_TO_NEAREST | CL_FP_ROUND_TO_ZERO | CL_FP_ROUND_TO_INF | CL_FP_INF_NAN | CL_FP_FMA; - - if (settings().singleFpDenorm_) { - info_.singleFPConfig_ |= CL_FP_DENORM; - } - - if (settings().checkExtension(ClKhrFp64)) { - info_.doubleFPConfig_ = info_.singleFPConfig_ | CL_FP_DENORM; - } - - if (settings().reportFMA_) { - info_.singleFPConfig_ |= CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT; - } - - info_.globalMemCacheLineSize_ = settings().cacheLineSize_; - info_.globalMemCacheSize_ = settings().cacheSize_; - if ((settings().cacheLineSize_ != 0) || (settings().cacheSize_ != 0)) { - info_.globalMemCacheType_ = CL_READ_WRITE_CACHE; - } else { - info_.globalMemCacheType_ = CL_NONE; - } - -#if defined(ATI_OS_LINUX) - info_.globalMemSize_ = - (static_cast(std::min(GPU_MAX_HEAP_SIZE, 100u)) * - // globalMemSize is the actual available size for app on Linux - // Because Linux base driver doesn't support paging - static_cast(memInfo.cardMemAvailableBytes + memInfo.cardExtMemAvailableBytes) / - 100u); -#else - info_.globalMemSize_ = (static_cast(std::min(GPU_MAX_HEAP_SIZE, 100u)) * - static_cast(calAttr.localRAM) / 100u) * - Mi; -#endif - int uswcPercentAvailable = (calAttr.uncachedRemoteRAM > 1536 && IS_WINDOWS) ? 75 : 50; - if (settings().apuSystem_) { - info_.globalMemSize_ += - (static_cast(calAttr.uncachedRemoteRAM) * Mi * uswcPercentAvailable) / 100; - } - -// We try to calculate the largest available memory size from -// the largest available block in either heap. In theory this -// should be the size we can actually allocate at application -// start. Note that it may not be a guarantee still as the -// application progresses. -#if defined(BRAHMA) && defined(ATI_BITS_64) - info_.maxMemAllocSize_ = - std::max(uint64_t(memInfo.cardMemAvailableBytes), uint64_t(memInfo.cardExtMemAvailableBytes)); -#else - info_.maxMemAllocSize_ = std::max(uint64_t(memInfo.cardLargestFreeBlockBytes), - uint64_t(memInfo.cardExtLargestFreeBlockBytes)); -#endif - - if (settings().apuSystem_) { - info_.maxMemAllocSize_ = std::max( - (static_cast(calAttr.uncachedRemoteRAM) * Mi * uswcPercentAvailable) / 100, - info_.maxMemAllocSize_); - } - info_.maxMemAllocSize_ = - uint64_t(info_.maxMemAllocSize_ * std::min(GPU_SINGLE_ALLOC_PERCENT, 100u) / 100u); - - //! \note Force max single allocation size. - //! 4GB limit for the blit kernels and 64 bit optimizations. - info_.maxMemAllocSize_ = - std::min(info_.maxMemAllocSize_, static_cast(settings().maxAllocSize_)); - - if (info_.maxMemAllocSize_ < uint64_t(128 * Mi)) { - LogError( - "We are unable to get a heap large enough to support the OpenCL minimum " - "requirement for FULL_PROFILE"); - } - - info_.maxMemAllocSize_ = std::max(uint64_t(128 * Mi), info_.maxMemAllocSize_); - - // Clamp max single alloc size to the globalMemSize since it's - // reduced by default - info_.maxMemAllocSize_ = std::min(info_.maxMemAllocSize_, info_.globalMemSize_); - - // We need to verify that we are not reporting more global memory - // that 4x single alloc - info_.globalMemSize_ = std::min(4 * info_.maxMemAllocSize_, info_.globalMemSize_); - - // Use 64 bit pointers - if (settings().use64BitPtr_) { - info_.addressBits_ = 64; - } else { - info_.addressBits_ = 32; - // Limit total size with 3GB for 32 bit - info_.globalMemSize_ = std::min(info_.globalMemSize_, uint64_t(3 * Gi)); - } - - // Alignment in BITS of the base address of any allocated memory object - static const size_t MemBaseAlignment = 256; - //! @note Force 256 bytes alignment, since currently - //! calAttr.surface_alignment returns 4KB. For pinned memory runtime - //! should be able to create a view with 256 bytes alignement - info_.memBaseAddrAlign_ = 8 * MemBaseAlignment; - - info_.preferredConstantBufferSize_ = 16 * Ki; - info_.maxConstantBufferSize_ = (settings().ciPlus_) ? info_.maxMemAllocSize_ : 64 * Ki; - info_.maxConstantArgs_ = MaxConstArguments; - - // Image support fields - if (settings().imageSupport_) { - info_.imageSupport_ = true; - info_.maxSamplers_ = MaxSamplers; - info_.maxReadImageArgs_ = MaxReadImage; - info_.maxWriteImageArgs_ = MaxWriteImage; - info_.image2DMaxWidth_ = maxTextureSize; - info_.image2DMaxHeight_ = maxTextureSize; - info_.image3DMaxWidth_ = std::min(2 * Ki, maxTextureSize); - info_.image3DMaxHeight_ = std::min(2 * Ki, maxTextureSize); - info_.image3DMaxDepth_ = std::min(2 * Ki, maxTextureSize); - - info_.imagePitchAlignment_ = 256; // XXX: 256 pixel pitch alignment for now - info_.imageBaseAddressAlignment_ = 256; // XXX: 256 byte base address alignment for now - - info_.bufferFromImageSupport_ = true; - } - - info_.errorCorrectionSupport_ = false; - - if (settings().apuSystem_) { - info_.hostUnifiedMemory_ = true; - } - - info_.profilingTimerResolution_ = 1; - info_.profilingTimerOffset_ = amd::Os::offsetToEpochNanos(); - info_.littleEndian_ = true; - info_.available_ = true; - info_.compilerAvailable_ = true; - info_.linkerAvailable_ = true; - - info_.executionCapabilities_ = CL_EXEC_KERNEL; - info_.preferredPlatformAtomicAlignment_ = 0; - info_.preferredGlobalAtomicAlignment_ = 0; - info_.preferredLocalAtomicAlignment_ = 0; - info_.queueProperties_ = CL_QUEUE_PROFILING_ENABLE; - - info_.platform_ = AMD_PLATFORM; - - ::strncpy(info_.name_, calName_, sizeof(info_.name_) - 1); - ::strncpy(info_.vendor_, "Advanced Micro Devices, Inc.", sizeof(info_.vendor_) - 1); - ::snprintf(info_.driverVersion_, sizeof(info_.driverVersion_) - 1, AMD_BUILD_STRING " (GSL)%s", - isOnline() ? "" : " [Offline]"); - - info_.profile_ = "FULL_PROFILE"; - if (settings().oclVersion_ >= OpenCL20) { - info_.version_ = "OpenCL 2.0 " AMD_PLATFORM_INFO; - info_.oclcVersion_ = "OpenCL C 2.0 "; - info_.spirVersions_ = "1.2"; - } else if (settings().oclVersion_ == OpenCL12) { - info_.version_ = "OpenCL 1.2 " AMD_PLATFORM_INFO; - info_.oclcVersion_ = "OpenCL C 1.2 "; - info_.spirVersions_ = "1.2"; - } else { - info_.version_ = "OpenCL 1.0 " AMD_PLATFORM_INFO; - info_.oclcVersion_ = "OpenCL C 1.0 "; - info_.spirVersions_ = ""; - LogError("Unknown version for support"); - } - - // Fill workgroup info size - info_.maxWorkGroupSize_ = settings().maxWorkGroupSize_; - info_.maxWorkItemSizes_[0] = info_.maxWorkGroupSize_; - info_.maxWorkItemSizes_[1] = info_.maxWorkGroupSize_; - info_.maxWorkItemSizes_[2] = info_.maxWorkGroupSize_; - info_.preferredWorkGroupSize_ = settings().preferredWorkGroupSize_; - - if (settings().hwLDSSize_ != 0) { - info_.localMemType_ = CL_LOCAL; - info_.localMemSize_ = settings().hwLDSSize_; - } else { - info_.localMemType_ = CL_GLOBAL; - info_.localMemSize_ = 16 * Ki; - } - - info_.extensions_ = getExtensionString(); - - ::strncpy(info_.driverStore_, calAttr.driverStore, sizeof(info_.driverStore_) - 1); - - // OpenCL1.2 device info fields - info_.builtInKernels_ = ""; - info_.imageMaxBufferSize_ = MaxImageBufferSize; - info_.image1DMaxWidth_ = maxTextureSize; - info_.imageMaxArraySize_ = MaxImageArraySize; - info_.preferredInteropUserSync_ = true; - info_.printfBufferSize_ = PrintfDbg::WorkitemDebugSize * info().maxWorkGroupSize_; - - if (settings().oclVersion_ >= OpenCL20) { - info_.svmCapabilities_ = (CL_DEVICE_SVM_COARSE_GRAIN_BUFFER | CL_DEVICE_SVM_FINE_GRAIN_BUFFER); - if (settings().svmAtomics_) { - info_.svmCapabilities_ |= CL_DEVICE_SVM_ATOMICS; - } - if (settings().svmFineGrainSystem_) { - info_.svmCapabilities_ |= CL_DEVICE_SVM_FINE_GRAIN_SYSTEM; - } - // OpenCL2.0 device info fields - info_.maxWriteImageArgs_ = MaxReadWriteImage; //!< For compatibility - info_.maxReadWriteImageArgs_ = MaxReadWriteImage; - - info_.maxPipePacketSize_ = info_.maxMemAllocSize_; - info_.maxPipeActiveReservations_ = 16; - info_.maxPipeArgs_ = 16; - - info_.queueOnDeviceProperties_ = - CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE | CL_QUEUE_PROFILING_ENABLE; - info_.queueOnDevicePreferredSize_ = 256 * Ki; - info_.queueOnDeviceMaxSize_ = 8 * Mi; - info_.maxOnDeviceQueues_ = 1; - info_.maxOnDeviceEvents_ = settings().numDeviceEvents_; - info_.globalVariablePreferredTotalSize_ = static_cast(info_.globalMemSize_); - //! \todo Remove % calculation. - //! Use 90% of max single alloc size. - //! Boards with max single alloc size around 4GB will fail allocations - info_.maxGlobalVariableSize_ = - static_cast(amd::alignDown(info_.maxMemAllocSize_ * 9 / 10, 256)); - } - - if (settings().checkExtension(ClAmdDeviceAttributeQuery)) { - ::strncpy(info_.boardName_, calAttr.boardName, sizeof(info_.boardName_) - 1); - - info_.deviceTopology_.pcie.type = CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD; - info_.deviceTopology_.pcie.bus = (calAttr.pciTopologyInformation & (0xFF << 8)) >> 8; - info_.deviceTopology_.pcie.device = (calAttr.pciTopologyInformation & (0x1F << 3)) >> 3; - info_.deviceTopology_.pcie.function = (calAttr.pciTopologyInformation & 0x07); - - info_.simdPerCU_ = isa().simdPerCU(); - info_.cuPerShaderArray_ = calAttr.numberOfCUsperShaderArray; - info_.simdWidth_ = isa().simdWidth(); - info_.simdInstructionWidth_ = isa().simdInstructionWidth(); - info_.wavefrontWidth_ = calAttr.wavefrontSize; - - info_.globalMemChannelBanks_ = calAttr.numMemBanks; - info_.globalMemChannelBankWidth_ = isa().memChannelBankWidth(); - info_.localMemSizePerCU_ = isa().localMemSizePerCU(); - info_.localMemBanks_ = isa().localMemBanks(); - - info_.numAsyncQueues_ = numComputeRings; - - info_.numRTQueues_ = numComputeRingsRT; - info_.numRTCUs_ = calAttr.maxRTCUs; - - info_.threadTraceEnable_ = settings().threadTraceEnable_; - - info_.pcieDeviceId_ = calAttr.pcieDeviceID; - info_.pcieRevisionId_ = calAttr.pcieRevisionID; - info_.maxThreadsPerCU_ = info_.wavefrontWidth_ * isa().simdPerCU() * 10; - } -} - -bool Device::Heap::create(Device& device) { - // Create global GPU heap - resource_ = new Memory(device, 0); - if (resource_ == NULL) { - return false; - } - - if (!resource_->create(Resource::Heap)) { - return false; - } - - baseAddress_ = resource_->gslResource()->getSurfaceAddress(); - return true; -} - -void Device::Engines::create(uint num, gslEngineDescriptor* desc, uint maxNumComputeRings) { - numComputeRings_ = 0; - numComputeRingsRT_ = 0; - numDmaEngines_ = 0; - - for (uint i = 0; i < num; ++i) { - desc_[desc[i].id] = desc[i]; - desc_[desc[i].id].priority = GSL_ENGINEPRIORITY_NEUTRAL; - - if (desc[i].id >= GSL_ENGINEID_COMPUTE0 && desc[i].id <= GSL_ENGINEID_COMPUTE7) { - numComputeRings_++; - } - - if (desc[i].id == GSL_ENGINEID_COMPUTE_RT) { - numComputeRingsRT_++; - } - if (desc[i].id == GSL_ENGINEID_COMPUTE_MEDIUM_PRIORITY) { - numComputeRingsRT_++; - } - - if (desc[i].id >= GSL_ENGINEID_DRMDMA0 && desc[i].id <= GSL_ENGINEID_DRMDMA1) { - numDmaEngines_++; - } - } - - numComputeRings_ = std::min(numComputeRings_, maxNumComputeRings); -} - -uint Device::Engines::getRequested(uint engines, gslEngineDescriptor* desc) const { - uint slot = 0; - for (uint i = 0; i < GSL_ENGINEID_MAX; ++i) { - if ((engines & getMask(static_cast(i))) && - (desc_[i].id == static_cast(i))) { - desc[slot] = desc_[i]; - engines &= ~getMask(static_cast(i)); - slot++; - } - } - return (engines == 0) ? slot : 0; -} - -Device::XferBuffers::~XferBuffers() { - // Destroy temporary buffer for reads - for (const auto& buf : freeBuffers_) { - // CPU optimization: unmap staging buffer just once - if (!buf->cal()->cardMemory_) { - buf->unmap(NULL); - } - delete buf; - } - freeBuffers_.clear(); -} - -bool Device::XferBuffers::create() { - bool result = false; - // Create a buffer object - Memory* xferBuf = new Memory(dev(), bufSize_); - - // Try to allocate memory for the transfer buffer - if ((NULL == xferBuf) || !xferBuf->create(type_)) { - delete xferBuf; - xferBuf = NULL; - LogError("Couldn't allocate a transfer buffer!"); - } else { - result = true; - freeBuffers_.push_back(xferBuf); - // CPU optimization: map staging buffer just once - if (!xferBuf->cal()->cardMemory_) { - xferBuf->map(NULL); - } - } - - return result; -} - -Memory& Device::XferBuffers::acquire() { - Memory* xferBuf = NULL; - size_t listSize; - - // Lock the operations with the staged buffer list - amd::ScopedLock l(lock_); - listSize = freeBuffers_.size(); - - // If the list is empty, then attempt to allocate a staged buffer - if (listSize == 0) { - // Allocate memory - xferBuf = new Memory(dev(), bufSize_); - - // Allocate memory for the transfer buffer - if ((NULL == xferBuf) || !xferBuf->create(type_)) { - delete xferBuf; - xferBuf = NULL; - LogError("Couldn't allocate a transfer buffer!"); - } else { - ++acquiredCnt_; - // CPU optimization: map staging buffer just once - if (!xferBuf->cal()->cardMemory_) { - xferBuf->map(NULL); - } - } - } - - if (xferBuf == NULL) { - xferBuf = *(freeBuffers_.begin()); - freeBuffers_.erase(freeBuffers_.begin()); - ++acquiredCnt_; - } - - return *xferBuf; -} - -void Device::XferBuffers::release(VirtualGPU& gpu, Memory& buffer) { - // Make sure buffer isn't busy on the current VirtualGPU, because - // the next aquire can come from different queue - buffer.wait(gpu); - // Lock the operations with the staged buffer list - amd::ScopedLock l(lock_); - freeBuffers_.push_back(&buffer); - --acquiredCnt_; -} - - -Device::ScopedLockVgpus::ScopedLockVgpus(const Device& dev) : dev_(dev) { - // Lock the virtual GPU list - dev_.vgpusAccess()->lock(); - - // Find all available virtual GPUs and lock them - // from the execution of commands - for (uint idx = 0; idx < dev_.vgpus().size(); ++idx) { - dev_.vgpus()[idx]->execution().lock(); - } -} - -Device::ScopedLockVgpus::~ScopedLockVgpus() { - // Find all available virtual GPUs and unlock them - // for the execution of commands - for (uint idx = 0; idx < dev_.vgpus().size(); ++idx) { - dev_.vgpus()[idx]->execution().unlock(); - } - - // Unock the virtual GPU list - dev_.vgpusAccess()->unlock(); -} - -Device::Device() - : NullDevice(), - CALGSLDevice(), - numOfVgpus_(0), - heap_(), - dummyPage_(NULL), - lockAsyncOps_(NULL), - lockAsyncOpsForInitHeap_(NULL), - vgpusAccess_(NULL), - scratchAlloc_(NULL), - mapCacheOps_(NULL), - xferRead_(NULL), - xferWrite_(NULL), - mapCache_(NULL), - resourceCache_(NULL), - heapInitComplete_(false), - xferQueue_(NULL), - globalScratchBuf_(NULL), - srdManager_(NULL) {} - -Device::~Device() { - // remove the HW debug manager - delete hwDebugMgr_; - hwDebugMgr_ = NULL; - - delete srdManager_; - - for (uint s = 0; s < scratch_.size(); ++s) { - delete scratch_[s]; - scratch_[s] = NULL; - } - - delete globalScratchBuf_; - globalScratchBuf_ = NULL; - - // Destroy transfer queue - delete xferQueue_; - - // Destroy blit program - delete blitProgram_; - - // Release cached map targets - for (uint i = 0; mapCache_ != NULL && i < mapCache_->size(); ++i) { - if ((*mapCache_)[i] != NULL) { - (*mapCache_)[i]->release(); - } - } - delete mapCache_; - - // Destroy temporary buffers for read/write - delete xferRead_; - delete xferWrite_; - - if (dummyPage_ != NULL) { - dummyPage_->release(); - } - - // Destroy resource cache - delete resourceCache_; - - delete lockAsyncOps_; - delete lockAsyncOpsForInitHeap_; - delete vgpusAccess_; - delete scratchAlloc_; - delete mapCacheOps_; - - if (context_ != NULL) { - context_->release(); - } - - // Close the active device - close(); -} - -extern const char* SchedulerSourceCode; - -bool Device::create(CALuint ordinal, CALuint numOfDevices) { - appProfile_.init(); - - bool smallMemSystem = false; - if (amd::Os::hostTotalPhysicalMemory() < OCL_SYSMEM_REQUIREMENT * Gi) { - smallMemSystem = true; - } - - bool noSVM = LP64_SWITCH(true, false) && !GPU_FORCE_OCL20_32BIT; - // Open GSL device - CALGSLDevice::OpenParams openData = {0}; - openData.enableHighPerformanceState = appProfile_.enableHighPerformanceState(); - openData.reportAsOCL12Device = (smallMemSystem || - appProfile_.reportAsOCL12Device() || - (OPENCL_VERSION < 200) || - noSVM); - openData.sclkThreshold = appProfile_.GetSclkThreshold().c_str(); - openData.downHysteresis = appProfile_.GetDownHysteresis().c_str(); - openData.upHysteresis = appProfile_.GetUpHysteresis().c_str(); - openData.powerLimit = appProfile_.GetPowerLimit().c_str(); - openData.mclkThreshold = appProfile_.GetMclkThreshold().c_str(); - openData.mclkUpHyst = appProfile_.GetMclkUpHyst().c_str(); - openData.mclkDownHyst = appProfile_.GetMclkDownHyst().c_str(); - - if (!open(ordinal, openData)) { - return false; - } - - // Update CAL target - calTarget_ = getAttribs().target; - - // XNACK should be set for PageMigration or IOMMUv2 support. - bool isXNACKSupported = false; - - // SRAMECC should be set for ecc protected GPRs. - bool isSRAMECCSupported = false; - - const amd::Isa* isa; - bool preferPal; - std::tie(isa, calMachine_, calName_, preferPal, std::ignore, std::ignore) = - findIsa(calTarget(), isSRAMECCSupported, isXNACKSupported); - - if ((calTarget() == CAL_TARGET_CARRIZO) && ASICREV_IS_CARRIZO_BRISTOL(getAttribs().asicRevision)) { - calName_ = "Bristol Ridge"; - } - - if (!isa) { - LogPrintfError("Unsupported CAL device #%d", calTarget()); - return false; - } - if (!isa->runtimeGslSupported()) { - LogPrintfError("Unsupported CAL device with ISA %s", isa->targetId()); - return false; - } - if ((GPU_ENABLE_PAL == 2) && isa->runtimePalSupported() && preferPal) { - LogPrintfError("Skipping as GPU_ENABLE_PAL=2 indicating to use PAL for CAL device %s", - isa->targetId()); - return false; - } - - if (!amd::Device::create(*isa)) { - LogPrintfError("Unable to setup device for CAL device %s", isa->targetId()); - return false; - } - - // Creates device settings - settings_ = new gpu::Settings(); - gpu::Settings* gpuSettings = reinterpret_cast(settings_); - if ((gpuSettings == NULL) || - !gpuSettings->create(getAttribs(), appProfile_.reportAsOCL12Device(), smallMemSystem)) { - return false; - } - - if (!ValidateHsail()) { - LogError("Hsail initialization failed!"); - return false; - } - - engines_.create(m_nEngines, m_engines, settings().numComputeRings_); - - amd::Context::Info info = {0}; - std::vector devices; - devices.push_back(this); - - // Create a dummy context - context_ = new amd::Context(devices, info); - if (context_ == NULL) { - return false; - } - - // Create the locks - lockAsyncOps_ = new amd::Monitor("Device Async Ops Lock", true); - if (NULL == lockAsyncOps_) { - return false; - } - - lockAsyncOpsForInitHeap_ = - new amd::Monitor("Async Ops Lock For Initialization of Heap Resource", true); - if (NULL == lockAsyncOpsForInitHeap_) { - return false; - } - - vgpusAccess_ = new amd::Monitor("Virtual GPU List Ops Lock", true); - if (NULL == vgpusAccess_) { - return false; - } - - scratchAlloc_ = new amd::Monitor("Scratch Allocation Lock", true); - if (NULL == scratchAlloc_) { - return false; - } - - mapCacheOps_ = new amd::Monitor("Map Cache Lock", true); - if (NULL == mapCacheOps_) { - return false; - } - - mapCache_ = new std::vector(); - if (mapCache_ == NULL) { - return false; - } - // Use just 1 entry by default for the map cache - mapCache_->push_back(NULL); - - size_t resourceCacheSize = settings().resourceCacheSize_; - -#ifdef DEBUG - std::stringstream message; - if (settings().remoteAlloc_) { - message << "Using *Remote* memory"; - } else { - message << "Using *Local* memory"; - } - - message << std::endl; - LogInfo(message.str().c_str()); -#endif // DEBUG - - // Create resource cache. - // \note Cache must be created before any resource creation to avoid NULL check - resourceCache_ = new ResourceCache(resourceCacheSize); - if (NULL == resourceCache_) { - return false; - } - - // Fill the device info structure - fillDeviceInfo(getAttribs(), getMemInfo(), static_cast(getMaxTextureSize()), - engines().numComputeRings(), engines().numComputeRingsRT()); - - if (NULL == hsaCompiler_) { - const char* library = getenv("HSA_COMPILER_LIBRARY"); - aclCompilerOptions opts = { - sizeof(aclCompilerOptions_0_8), library, NULL, NULL, NULL, NULL, NULL, AMD_OCL_SC_LIB}; - // Initialize the compiler handle - acl_error error; - hsaCompiler_ = amd::Hsail::CompilerInit(&opts, &error); - if (error != ACL_SUCCESS) { - LogError("Error initializing the compiler"); - return false; - } - } - - // Allocate SRD manager - srdManager_ = new SrdManager(*this, std::max(HsaImageObjectSize, HsaSamplerObjectSize), 64 * Ki); - if (srdManager_ == NULL) { - return false; - } - - // create the HW debug manager if needed - if (settings().enableHwDebug_) { - hwDebugMgr_ = new GpuDebugManager(this); - } - - return true; -} - -bool Device::initializeHeapResources() { - amd::ScopedLock k(lockAsyncOpsForInitHeap_); - if (!heapInitComplete_) { - heapInitComplete_ = true; - - PerformFullInitialization(); - - uint numComputeRings = engines_.numComputeRings() + engines_.numComputeRingsRT(); - scratch_.resize((settings().useSingleScratch_) ? 1 : (numComputeRings ? numComputeRings : 1)); - - // Initialize the number of mem object for the scratch buffer - for (uint s = 0; s < scratch_.size(); ++s) { - scratch_[s] = new ScratchBuffer(); - if (NULL == scratch_[s]) { - return false; - } - } - - // Complete initialization of the heap and other buffers - if (!heap_.create(*this)) { - LogError("Failed GPU heap creation"); - return false; - } - - size_t dummySize = amd::Os::pageSize(); - - // Allocate a dummy page for NULL pointer processing - dummyPage_ = new (*context_) amd::Buffer(*context_, 0, dummySize); - if ((dummyPage_ != NULL) && !dummyPage_->create()) { - dummyPage_->release(); - return false; - } - - Memory* devMemory = reinterpret_cast(dummyPage_->getDeviceMemory(*this)); - if (devMemory == NULL) { - // Release memory - dummyPage_->release(); - dummyPage_ = NULL; - return false; - } - - if (settings().stagedXferSize_ != 0) { - // Initialize staged write buffers - if (settings().stagedXferWrite_) { - Resource::MemoryType type = Resource::RemoteUSWC; - xferWrite_ = new XferBuffers(*this, type, amd::alignUp(settings().stagedXferSize_, 4 * Ki)); - if ((xferWrite_ == NULL) || !xferWrite_->create()) { - LogError("Couldn't allocate transfer buffer objects for read"); - return false; - } - } - - // Initialize staged read buffers - if (settings().stagedXferRead_) { - xferRead_ = new XferBuffers(*this, Resource::Remote, - amd::alignUp(settings().stagedXferSize_, 4 * Ki)); - if ((xferRead_ == NULL) || !xferRead_->create()) { - LogError("Couldn't allocate transfer buffer objects for write"); - return false; - } - } - } - - // Delay compilation due to brig_loader memory allocation - if (settings().ciPlus_) { - std::string CL20extraBlits; - const std::string ocl20; - if (settings().oclVersion_ >= OpenCL20) { - CL20extraBlits = SchedulerSourceCode; - ocl20 = "-cl-std=CL2.0"; - } - blitProgram_ = new BlitProgram(context_); - // Create blit programs - if (blitProgram_ == NULL || !blitProgram_->create(this, CL20extraBlits, ocl20)) { - delete blitProgram_; - blitProgram_ = NULL; - LogError("Couldn't create blit kernels!"); - return false; - } - } - - // Create a synchronized transfer queue - xferQueue_ = new VirtualGPU(*this); - if (!(xferQueue_ && xferQueue_->create(false))) { - delete xferQueue_; - xferQueue_ = NULL; - } - if (NULL == xferQueue_) { - LogError("Couldn't create the device transfer manager!"); - return false; - } - xferQueue_->enableSyncedBlit(); - } - return true; -} - -device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) { - bool profiling = false; - uint rtCUs = amd::CommandQueue::RealTimeDisabled; - uint deviceQueueSize = 0; - - if (queue != NULL) { - bool interopQueue = false; - profiling = queue->properties().test(CL_QUEUE_PROFILING_ENABLE); - if (queue->asHostQueue() != NULL) { - interopQueue = (0 != (queue->context().info().flags_ & - (amd::Context::GLDeviceKhr | amd::Context::D3D10DeviceKhr | - amd::Context::D3D11DeviceKhr))); - rtCUs = queue->rtCUs(); - } else if (queue->asDeviceQueue() != NULL) { - deviceQueueSize = queue->asDeviceQueue()->size(); - } - } - - // Not safe to add a queue. So lock the device - amd::ScopedLock k(lockAsyncOps()); - amd::ScopedLock lock(vgpusAccess()); - - // Initialization of heap and other resources occur during the command queue creation time. - if (!initializeHeapResources()) { - return NULL; - } - - VirtualGPU* vgpu = new VirtualGPU(*this); - if (vgpu && vgpu->create(profiling, rtCUs, deviceQueueSize, queue->priority())) { - return vgpu; - } else { - delete vgpu; - return NULL; - } -} - -device::Program* Device::createProgram(amd::Program& owner, amd::option::Options* options) { - if (isHsailProgram(options)) { - return new HSAILProgram(*this, owner); - } - return new Program(*this, owner); -} - -//! Requested devices list as configured by the GPU_DEVICE_ORDINAL -typedef std::unordered_map requestedDevices_t; - -//! Parses the requested list of devices to be exposed to the user. -static void parseRequestedDeviceList(requestedDevices_t& requestedDevices) { - int requestedDeviceCount = 0; - const char* requestedDeviceList = GPU_DEVICE_ORDINAL; - - char* pch = strtok(const_cast(requestedDeviceList), ","); - while (pch != NULL) { - bool deviceIdValid = true; - int currentDeviceIndex = atoi(pch); - // Validate device index. - for (size_t i = 0; i < strlen(pch); i++) { - if (!isdigit(pch[i])) { - deviceIdValid = false; - break; - } - } - if (currentDeviceIndex < 0) { - deviceIdValid = false; - } - // Get next token. - pch = strtok(NULL, ","); - if (!deviceIdValid) { - continue; - } - - // Requested device is valid. - requestedDevices[currentDeviceIndex] = true; - } -} - -bool Device::init() { - CALuint numDevices = 0; - bool useDeviceList = false; - requestedDevices_t requestedDevices; - - hsaCompiler_ = NULL; - compiler_ = NULL; - -#if defined(_WIN32) && !defined(_WIN64) - // @toto: FIXME: remove this when CAL is fixed!!! - unsigned int old, ignored; - _controlfp_s(&old, 0, 0); -#endif // _WIN32 && !_WIN64 - // FIXME_lmoriche: needs cleanup - osInit(); -#if defined(_WIN32) -// osAssertSetStyle(OSASSERT_STYLE_LOGANDEXIT); -#endif // WIN32 - - gslInit(); - -#if defined(_WIN32) && !defined(_WIN64) - _controlfp_s(&ignored, old, _MCW_RC | _MCW_PC); -#endif // _WIN32 && !_WIN64 - - // Get the total number of active devices - // Count up all the devices in the system. - numDevices = gsAdaptor::enumerateAdaptors(); - - const char* selectDeviceByName = NULL; - if (!flagIsDefault(GPU_DEVICE_ORDINAL)) { - useDeviceList = true; - parseRequestedDeviceList(requestedDevices); - } - - // Loop through all active devices and initialize the device info structure - for (CALuint ordinal = 0; ordinal < numDevices; ++ordinal) { - // Create the GPU device object - Device* d = new Device(); - bool result = (NULL != d) && d->create(ordinal, numDevices); - if (useDeviceList) { - result &= (requestedDevices.find(ordinal) != requestedDevices.end()); - } - if (result) { - d->registerDevice(); - } else { - delete d; - } - } - return true; -} - -void Device::tearDown() { - osExit(); - gslExit(); - amd::Hsail::CompilerFini(compiler_); - if (hsaCompiler_ != NULL) { - amd::Hsail::CompilerFini(hsaCompiler_); - } -} - -gpu::Memory* Device::getGpuMemory(amd::Memory* mem) const { - return static_cast(mem->getDeviceMemory(*this)); -} - -const device::BlitManager& Device::xferMgr() const { return xferQueue_->blitMgr(); } - -CalFormat Device::getCalFormat(const amd::Image::Format& format) const { - // Find CAL format - for (uint i = 0; i < sizeof(MemoryFormatMap) / sizeof(MemoryFormat); ++i) { - if ((format.image_channel_data_type == MemoryFormatMap[i].clFormat_.image_channel_data_type) && - (format.image_channel_order == MemoryFormatMap[i].clFormat_.image_channel_order)) { - return MemoryFormatMap[i].calFormat_; - } - } - osAssert(0 && "We didn't find CAL resource format!"); - return MemoryFormatMap[0].calFormat_; -} - -amd::Image::Format Device::getOclFormat(const CalFormat& format) const { - // Find CL format - for (uint i = 0; i < sizeof(MemoryFormatMap) / sizeof(MemoryFormat); ++i) { - if ((format.type_ == MemoryFormatMap[i].calFormat_.type_) && - (format.channelOrder_ == MemoryFormatMap[i].calFormat_.channelOrder_)) { - return MemoryFormatMap[i].clFormat_; - } - } - osAssert(0 && "We didn't find OCL resource format!"); - return MemoryFormatMap[0].clFormat_; -} - -// Create buffer without an owner (merge common code with createBuffer() ?) -gpu::Memory* Device::createScratchBuffer(size_t size) const { - - // Create a memory object - Memory* gpuMemory = new gpu::Memory(*this, size); - if (NULL == gpuMemory || !gpuMemory->create(Resource::Local)) { - delete gpuMemory; - gpuMemory = NULL; - } - - return gpuMemory; -} - -gpu::Memory* Device::createBuffer(amd::Memory& owner, bool directAccess) const { - size_t size = owner.getSize(); - gpu::Memory* gpuMemory; - - // Create resource - bool result = false; - - if (owner.getType() == CL_MEM_OBJECT_PIPE) { - // directAccess isnt needed as Pipes shouldnt be host accessible for GPU - directAccess = false; - } - - if (NULL != owner.parent()) { - gpu::Memory* gpuParent = getGpuMemory(owner.parent()); - if (NULL == gpuParent) { - LogError("Can't get the owner object for subbuffer allocation"); - return NULL; - } - - if (nullptr != owner.parent()->getSvmPtr()) { - amd::Memory* amdParent = owner.parent(); - { - // Lock memory object, so only one commitment will occur - amd::ScopedLock lock(amdParent->lockMemoryOps()); - amdParent->commitSvmMemory(); - amdParent->setHostMem(amdParent->getSvmPtr()); - } - // Ignore a possible pinning error. Runtime will fallback to SW emulation - // bool ok = gpuParent->pinSystemMemory( - // amdParent->getHostMem(), amdParent->getSize()); - } - return gpuParent->createBufferView(owner); - } - - Resource::MemoryType type = - (owner.forceSysMemAlloc() || (owner.getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) - ? Resource::Remote - : Resource::Local; - - // Check if runtime can force a tiny buffer into USWC memory - if ((size <= (GPU_MAX_REMOTE_MEM_SIZE * Ki)) && (type == Resource::Local) && - (owner.getMemFlags() & CL_MEM_READ_ONLY)) { - type = Resource::RemoteUSWC; - } - - if (owner.getMemFlags() & CL_MEM_BUS_ADDRESSABLE_AMD) { - type = Resource::BusAddressable; - } else if (owner.getMemFlags() & CL_MEM_EXTERNAL_PHYSICAL_AMD) { - type = Resource::ExternalPhysical; - } - - // Use direct access if it's possible - bool remoteAlloc = false; - // Internal means VirtualDevice!=NULL - bool internalAlloc = - ((owner.getMemFlags() & CL_MEM_USE_HOST_PTR) && (owner.getVirtualDevice() != NULL)) ? true - : false; - - // Create a memory object - gpuMemory = new gpu::Buffer(*this, owner, owner.getSize()); - if (NULL == gpuMemory) { - return NULL; - } - - // Check if owner is interop memory - if (owner.isInterop()) { - result = gpuMemory->createInterop(Memory::InteropDirectAccess); - } else if (owner.getMemFlags() & CL_MEM_USE_PERSISTENT_MEM_AMD) { - // Attempt to allocate from persistent heap - result = gpuMemory->create(Resource::Persistent); - } else if (directAccess || (type == Resource::Remote)) { - // Check for system memory allocations - if ((owner.getMemFlags() & (CL_MEM_ALLOC_HOST_PTR | CL_MEM_USE_HOST_PTR)) || - (settings().remoteAlloc_)) { - // Allocate remote memory if AHP allocation and context has just 1 device - if ((owner.getMemFlags() & CL_MEM_ALLOC_HOST_PTR) && - (owner.getContext().devices().size() == 1)) { - if (owner.getMemFlags() & - (CL_MEM_READ_ONLY | CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_NO_ACCESS)) { - // GPU will be reading from this host memory buffer, - // so assume Host write into it - type = Resource::RemoteUSWC; - remoteAlloc = true; - } - } - // Make sure owner has a valid hostmem pointer and it's not COPY - if (!remoteAlloc && (owner.getHostMem() != NULL)) { - Resource::PinnedParams params; - params.owner_ = &owner; - params.gpu_ = reinterpret_cast(owner.getVirtualDevice()); - - params.hostMemRef_ = owner.getHostMemRef(); - params.size_ = owner.getHostMemRef()->size(); - if (0 == params.size_) { - params.size_ = owner.getSize(); - } - // Create memory object - result = gpuMemory->create(Resource::Pinned, ¶ms); - - // If direct access failed - if (!result) { - // Don't use cached allocation - // if size is biger than max single alloc - if (owner.getSize() > info().maxMemAllocSize_) { - delete gpuMemory; - return NULL; - } - } - } - } - } - - if (!result && - // Make sure it's not internal alloc - !internalAlloc) { - Resource::CreateParams params; - params.owner_ = &owner; - params.gpu_ = static_cast(owner.getVirtualDevice()); - - // Create memory object - result = gpuMemory->create(type, ¶ms); - - // If allocation was successful - if (result) { - // Initialize if the memory is a pipe object - if (owner.getType() == CL_MEM_OBJECT_PIPE) { - // Pipe initialize in order read_idx, write_idx, end_idx. Refer clk_pipe_t structure. - // Init with 3 DWORDS for 32bit addressing and 6 DWORDS for 64bit - size_t pipeInit[3] = {0, 0, owner.asPipe()->getMaxNumPackets()}; - gpuMemory->writeRawData(*xferQueue_, sizeof(pipeInit), pipeInit, true); - } - // If memory has direct access from host, then get CPU address - if (gpuMemory->isHostMemDirectAccess() && (type != Resource::ExternalPhysical)) { - void* address = gpuMemory->map(NULL); - if (address != NULL) { - // Copy saved memory - // Note: UHP is an optional check if pinning failed and sysmem alloc was forced - if (owner.getMemFlags() & (CL_MEM_COPY_HOST_PTR | CL_MEM_USE_HOST_PTR)) { - memcpy(address, owner.getHostMem(), owner.getSize()); - } - // It should be safe to change the host memory pointer, - // because it's lock protected from the upper caller - owner.setHostMem(address); - } else { - result = false; - } - } - // An optimization for CHP. Copy memory and destroy sysmem allocation - else if ((gpuMemory->memoryType() != Resource::Pinned) && - (owner.getMemFlags() & CL_MEM_COPY_HOST_PTR) && - (owner.getContext().devices().size() == 1)) { - amd::Coord3D origin(0, 0, 0); - amd::Coord3D region(owner.getSize()); - static const bool Entire = true; - if (xferMgr().writeBuffer(owner.getHostMem(), *gpuMemory, origin, region, Entire)) { - // Clear CHP memory - owner.setHostMem(NULL); - } - } - } - } - - if (!result) { - delete gpuMemory; - return NULL; - } - - return gpuMemory; -} - -gpu::Memory* Device::createImage(amd::Memory& owner, bool directAccess) const { - amd::Image& image = *owner.asImage(); - gpu::Memory* gpuImage = NULL; - CalFormat format = getCalFormat(image.getImageFormat()); - - if ((NULL != owner.parent()) && (owner.parent()->asImage() != NULL)) { - device::Memory* devParent = owner.parent()->getDeviceMemory(*this); - if (NULL == devParent) { - LogError("Can't get the owner object for image view allocation"); - return NULL; - } - // Create a view on the specified device - gpuImage = (gpu::Memory*)createView(owner, *devParent); - if ((NULL != gpuImage) && (gpuImage->owner() != NULL)) { - gpuImage->owner()->setHostMem((address)(owner.parent()->getHostMem()) + - gpuImage->owner()->getOrigin()); - } - return gpuImage; - } - - gpuImage = - new gpu::Image(*this, owner, image.getWidth(), image.getHeight(), image.getDepth(), - format.type_, format.channelOrder_, image.getType(), image.getMipLevels()); - - // Create resource - if (NULL != gpuImage) { - const bool imageBuffer = ((owner.getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER) || - ((owner.getType() == CL_MEM_OBJECT_IMAGE2D) && - (owner.parent() != NULL) && (owner.parent()->asBuffer() != NULL))); - bool result = false; - - // Check if owner is interop memory - if (owner.isInterop()) { - result = gpuImage->createInterop(Memory::InteropDirectAccess); - } else if (imageBuffer) { - Resource::ImageBufferParams params; - gpu::Memory* buffer = reinterpret_cast(image.parent()->getDeviceMemory(*this)); - if (buffer == NULL) { - LogError("Buffer creation for ImageBuffer failed!"); - delete gpuImage; - return NULL; - } - params.owner_ = &owner; - params.resource_ = buffer; - params.memory_ = buffer; - - // Create memory object - result = gpuImage->create(Resource::ImageBuffer, ¶ms); - } else if (directAccess && (owner.getMemFlags() & CL_MEM_ALLOC_HOST_PTR)) { - Resource::PinnedParams params; - params.owner_ = &owner; - params.hostMemRef_ = owner.getHostMemRef(); - params.size_ = owner.getHostMemRef()->size(); - - // Create memory object - result = gpuImage->create(Resource::Pinned, ¶ms); - } - - if (!result && !owner.isInterop()) { - if (owner.getMemFlags() & CL_MEM_USE_PERSISTENT_MEM_AMD) { - // Attempt to allocate from persistent heap - result = gpuImage->create(Resource::Persistent); - } else { - Resource::MemoryType type = - (owner.forceSysMemAlloc()) ? Resource::RemoteUSWC : Resource::Local; - // Create memory object - result = gpuImage->create(type); - } - } - - if (!result) { - delete gpuImage; - return NULL; - } else if ((gpuImage->memoryType() != Resource::Pinned) && - (owner.getMemFlags() & CL_MEM_COPY_HOST_PTR) && - (owner.getContext().devices().size() == 1)) { - // Ignore copy for image1D_buffer, since it was already done for buffer - if (imageBuffer) { - // Clear CHP memory - owner.setHostMem(NULL); - } else { - amd::Coord3D origin(0, 0, 0); - static const bool Entire = true; - if (xferMgr().writeImage(owner.getHostMem(), *gpuImage, origin, image.getRegion(), 0, 0, - Entire)) { - // Clear CHP memory - owner.setHostMem(NULL); - } - } - } - - if (result) { - gslMemObject temp = gpuImage->gslResource(); - size_t bytePitch = gpuImage->elementSize() * temp->getPitch(); - image.setBytePitch(bytePitch); - } - } - - return gpuImage; -} - -//! Allocates cache memory on the card -device::Memory* Device::createMemory(amd::Memory& owner) const { - bool directAccess = false; - gpu::Memory* memory = NULL; - - if (owner.asBuffer()) { - directAccess = (settings().hostMemDirectAccess_ & Settings::HostMemBuffer) ? true : false; - memory = createBuffer(owner, directAccess); - } else if (owner.asImage()) { - directAccess = (settings().hostMemDirectAccess_ & Settings::HostMemImage) ? true : false; - memory = createImage(owner, directAccess); - } else { - LogError("Unknown memory type!"); - } - - // Attempt to pin system memory if runtime didn't use direct access - if ((memory != NULL) && (memory->memoryType() != Resource::Pinned) && - (memory->memoryType() != Resource::Remote) && - (memory->memoryType() != Resource::RemoteUSWC) && - (memory->memoryType() != Resource::ExternalPhysical) && - ((owner.getHostMem() != NULL) || - ((NULL != owner.parent()) && (owner.getHostMem() != NULL)))) { - bool ok = memory->pinSystemMemory(owner.getHostMem(), (owner.getHostMemRef()->size()) - ? owner.getHostMemRef()->size() - : owner.getSize()); - //! \note: Ignore the pinning result for now - } - - return memory; -} - -bool Device::createSampler(const amd::Sampler& owner, device::Sampler** sampler) const { - *sampler = NULL; - Sampler* gpuSampler = new Sampler(*this); - if ((NULL == gpuSampler) || !gpuSampler->create(owner)) { - delete gpuSampler; - return false; - } - *sampler = gpuSampler; - return true; -} - -device::Memory* Device::createView(amd::Memory& owner, const device::Memory& parent) const { - assert((owner.asImage() != NULL) && "View supports images only"); - const amd::Image& image = *owner.asImage(); - CalFormat format = getCalFormat(image.getImageFormat()); - - gpu::Memory* gpuImage = - new gpu::Image(*this, owner, image.getWidth(), image.getHeight(), image.getDepth(), - format.type_, format.channelOrder_, image.getType(), image.getMipLevels()); - - // Create resource - if (NULL != gpuImage) { - Resource::ImageViewParams params; - const gpu::Memory& gpuMem = static_cast(parent); - - params.owner_ = &owner; - params.level_ = image.getBaseMipLevel(); - params.layer_ = 0; - params.resource_ = &gpuMem; - params.gpu_ = reinterpret_cast(owner.getVirtualDevice()); - params.memory_ = &gpuMem; - - // Create memory object - bool result = gpuImage->create(Resource::ImageView, ¶ms); - if (!result) { - delete gpuImage; - return NULL; - } - } - - return gpuImage; -} - - -//! Attempt to bind with external graphics API's device/context -bool Device::bindExternalDevice(uint flags, void* const pDevice[], void* pContext, - bool validateOnly) { - assert(pDevice); - - if (flags & amd::Context::Flags::GLDeviceKhr) { - // There is no need to perform full initialization here - // if the GSLDevice is still uninitialized. - // Only adapter initialization is required to validate - // GL interoperability. - PerformAdapterInitialization(validateOnly); - - // Attempt to associate GSL-OGL - if (!glAssociate((CALvoid*)pContext, pDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx])) { - CloseInitializedAdapter(validateOnly); - LogError("Failed gslGLAssociate()"); - return false; - } - - CloseInitializedAdapter(validateOnly); - } - -#ifdef _WIN32 - if (flags & amd::Context::Flags::D3D10DeviceKhr) { - // There is no need to perform full initialization here - // if the GSLDevice is still uninitialized. - // Only adapter initialization is required - // to validate D3D10 interoperability. - PerformAdapterInitialization(validateOnly); - - // Associate GSL-D3D - if (!associateD3D10Device(reinterpret_cast( - pDevice[amd::Context::DeviceFlagIdx::D3D10DeviceKhrIdx]))) { - CloseInitializedAdapter(validateOnly); - LogError("Failed gslD3D10Associate()"); - return false; - } - - CloseInitializedAdapter(validateOnly); - } - - if (flags & amd::Context::Flags::D3D11DeviceKhr) { - // There is no need to perform full initialization here - // if the GSLDevice is still uninitialized. - // Only adapter initialization is required to validate - // D3D11 interoperability. - PerformAdapterInitialization(validateOnly); - - // Associate GSL-D3D - if (!associateD3D11Device(reinterpret_cast( - pDevice[amd::Context::DeviceFlagIdx::D3D11DeviceKhrIdx]))) { - CloseInitializedAdapter(validateOnly); - LogError("Failed gslD3D11Associate()"); - return false; - } - - CloseInitializedAdapter(validateOnly); - } - - if (flags & amd::Context::Flags::D3D9DeviceKhr) { - PerformAdapterInitialization(validateOnly); - - // Associate GSL-D3D - if (!associateD3D9Device(reinterpret_cast( - pDevice[amd::Context::DeviceFlagIdx::D3D9DeviceKhrIdx]))) { - CloseInitializedAdapter(validateOnly); - LogWarning("D3D9<->OpenCL adapter mismatch or D3D9Associate() failure"); - return false; - } - - CloseInitializedAdapter(validateOnly); - } - - if (flags & amd::Context::Flags::D3D9DeviceEXKhr) { - PerformAdapterInitialization(validateOnly); - - // Associate GSL-D3D - if (!associateD3D9Device(reinterpret_cast( - pDevice[amd::Context::DeviceFlagIdx::D3D9DeviceEXKhrIdx]))) { - CloseInitializedAdapter(validateOnly); - LogWarning("D3D9<->OpenCL adapter mismatch or D3D9Associate() failure"); - return false; - } - - CloseInitializedAdapter(validateOnly); - } - - if (flags & amd::Context::Flags::D3D9DeviceVAKhr) { - } -#endif //_WIN32 - return true; -} - -bool Device::unbindExternalDevice(uint flags, void* const pDevice[], void* pContext, - bool validateOnly) { - if ((flags & amd::Context::Flags::GLDeviceKhr) == 0) { - return true; - } - - void* glDevice = pDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx]; - if (glDevice != NULL) { - // Dissociate GSL-OGL - if (true != glDissociate(pContext, glDevice)) { - if (validateOnly) { - LogWarning("Failed gslGLDiassociate()"); - } - return false; - } - } - return true; -} - -bool Device::globalFreeMemory(size_t* freeMemory) const { - const uint TotalFreeMemory = 0; - const uint LargestFreeBlock = 1; - - // Initialization of heap and other resources because getMemInfo needs it. - if (!(const_cast(this)->initializeHeapResources())) { - return false; - } - - gslMemInfo memInfo = { 0 }; - gslCtx()->getMemInfo(&memInfo, GSL_MEMINFO_BASIC); - - // Fill free memory info - freeMemory[TotalFreeMemory] = (memInfo.cardMemAvailableBytes + memInfo.cardExtMemAvailableBytes + - resourceCache().lclCacheSize()) / Ki; - freeMemory[LargestFreeBlock] = - std::max(memInfo.cardLargestFreeBlockBytes, memInfo.cardExtLargestFreeBlockBytes) / Ki; - if (settings().apuSystem_) { - uint64_t sysMem = 0; - if ((memInfo.agpMemAvailableBytes + resourceCache().cacheSize()) > resourceCache().lclCacheSize()) { - sysMem = (memInfo.agpMemAvailableBytes + resourceCache().cacheSize()) - resourceCache().lclCacheSize(); - } - sysMem /= Ki; - freeMemory[TotalFreeMemory] += sysMem; - - if (settings().viPlus_) { - // for viPlus_, OCL is using remote instead remoteUSWC to avoid extra copy - freeMemory[LargestFreeBlock] += memInfo.agpCacheableLargestFreeBlockBytes / Ki; - } else { - freeMemory[LargestFreeBlock] += memInfo.agpLargestFreeBlockBytes / Ki; - } - } - return true; -} - -amd::Memory* Device::findMapTarget(size_t size) const { - // Must be serialised for access - amd::ScopedLock lk(*mapCacheOps_); - - amd::Memory* map = NULL; - size_t minSize = 0; - size_t maxSize = 0; - uint mapId = mapCache_->size(); - uint releaseId = mapCache_->size(); - - // Find if the list has a map target of appropriate size - for (uint i = 0; i < mapCache_->size(); i++) { - if ((*mapCache_)[i] != NULL) { - // Requested size is smaller than the entry size - if (size < (*mapCache_)[i]->getSize()) { - if ((minSize == 0) || (minSize > (*mapCache_)[i]->getSize())) { - minSize = (*mapCache_)[i]->getSize(); - mapId = i; - } - } - // Requeted size matches the entry size - else if (size == (*mapCache_)[i]->getSize()) { - mapId = i; - break; - } else { - // Find the biggest map target in the list - if (maxSize < (*mapCache_)[i]->getSize()) { - maxSize = (*mapCache_)[i]->getSize(); - releaseId = i; - } - } - } - } - - // Check if we found any map target - if (mapId < mapCache_->size()) { - map = (*mapCache_)[mapId]; - (*mapCache_)[mapId] = NULL; - Memory* gpuMemory = reinterpret_cast(map->getDeviceMemory(*this)); - - // Get the base pointer for the map resource - if ((gpuMemory == NULL) || (NULL == gpuMemory->map(NULL))) { - (*mapCache_)[mapId]->release(); - map = NULL; - } - } - // If cache is full, then release the biggest map target - else if (releaseId < mapCache_->size()) { - (*mapCache_)[releaseId]->release(); - (*mapCache_)[releaseId] = NULL; - } - - return map; -} - -bool Device::addMapTarget(amd::Memory* memory) const { - // Must be serialised for access - amd::ScopedLock lk(*mapCacheOps_); - - // the svm memory shouldn't be cached - if (!memory->canBeCached()) { - return false; - } - // Find if the list has a map target of appropriate size - for (uint i = 0; i < mapCache_->size(); ++i) { - if ((*mapCache_)[i] == NULL) { - (*mapCache_)[i] = memory; - return true; - } - } - - // Add a new entry - mapCache_->push_back(memory); - - return true; -} - -Device::ScratchBuffer::~ScratchBuffer() { destroyMemory(); } - -void Device::ScratchBuffer::destroyMemory() { - // Release memory object - delete memObj_; - memObj_ = NULL; -} - -bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu) { - if (regNum > 0) { - // Serialize the scratch buffer allocation code - amd::ScopedLock lk(*scratchAlloc_); - uint sb = vgpu->hwRing(); - - static const uint WaveSizeLimit = ((1 << 21) - 256); - const uint threadSizeLimit = WaveSizeLimit / getAttribs().wavefrontSize; - if (regNum > threadSizeLimit) { - LogError("Requested private memory is bigger than HW supports!"); - regNum = threadSizeLimit; - } - - // Check if the current buffer isn't big enough - if (regNum > scratch_[sb]->regNum_) { - // Stall all command queues, since runtime will reallocate memory - ScopedLockVgpus lock(*this); - - scratch_[sb]->regNum_ = regNum; - uint64_t size = 0; - uint64_t offset = 0; - - // Destroy all views - for (uint s = 0; s < scratch_.size(); ++s) { - ScratchBuffer* scratchBuf = scratch_[s]; - if (scratchBuf->regNum_ > 0) { - scratchBuf->destroyMemory(); - // Calculate the size of the scratch buffer for a queue - scratchBuf->size_ = calcScratchBufferSize(scratchBuf->regNum_); - scratchBuf->size_ = std::min(scratchBuf->size_, info().maxMemAllocSize_); - scratchBuf->size_ = std::min(scratchBuf->size_, uint64_t(3 * Gi)); - scratchBuf->size_ = amd::alignUp(scratchBuf->size_, 0xFFFF); - scratchBuf->offset_ = offset; - size += scratchBuf->size_; - offset += scratchBuf->size_; - } - } - - delete globalScratchBuf_; - - // Allocate new buffer. - globalScratchBuf_ = new gpu::Memory(*this, static_cast(size)); - if ((globalScratchBuf_ == NULL) || !globalScratchBuf_->create(Resource::Scratch)) { - LogError("Couldn't allocate scratch memory"); - for (uint s = 0; s < scratch_.size(); ++s) { - scratch_[s]->regNum_ = 0; - } - return false; - } - - for (uint s = 0; s < scratch_.size(); ++s) { - // Loop through all memory objects and reallocate them - if (scratch_[s]->regNum_ > 0) { - // Allocate new buffer - scratch_[s]->memObj_ = new gpu::Memory(*this, scratch_[s]->size_); - Resource::ViewParams view; - view.resource_ = globalScratchBuf_; - view.offset_ = scratch_[s]->offset_; - view.size_ = scratch_[s]->size_; - if ((scratch_[s]->memObj_ == NULL) || - !scratch_[s]->memObj_->create(Resource::View, &view)) { - LogError("Couldn't allocate a scratch view"); - delete scratch_[s]->memObj_; - scratch_[s]->regNum_ = 0; - return false; - } - } - } - } - } - return true; -} - -bool Device::validateKernel( - const amd::Kernel& kernel, const device::VirtualDevice* vdev, bool coop_groups) { - // Find the number of scratch registers used in the kernel - const device::Kernel* devKernel = kernel.getDeviceKernel(*this); - uint regNum = static_cast(devKernel->workGroupInfo()->scratchRegs_); - const VirtualGPU* vgpu = static_cast(vdev); - - if (!allocScratch(regNum, vgpu)) { - return false; - } - - if (devKernel->hsa()) { - const HSAILKernel* hsaKernel = static_cast(devKernel); - if (hsaKernel->dynamicParallelism()) { - amd::DeviceQueue* defQueue = kernel.program().context().defDeviceQueue(*this); - if (defQueue != NULL) { - vgpu = static_cast(defQueue->vDev()); - if (!allocScratch(hsaKernel->prog().maxScratchRegs(), vgpu)) { - return false; - } - } else { - return false; - } - } - } - - return true; -} - -void Device::destroyScratchBuffers() { - if (globalScratchBuf_ != NULL) { - for (uint s = 0; s < scratch_.size(); ++s) { - scratch_[s]->destroyMemory(); - scratch_[s]->regNum_ = 0; - } - delete globalScratchBuf_; - globalScratchBuf_ = NULL; - } -} - -void Device::fillHwSampler(uint32_t state, void* hwState, uint32_t hwStateSize, uint32_t mipFilter, - float minLod, float maxLod) const { - // All GSL sampler's parameters are in floats - uint32_t gslAddress = GSL_CLAMP_TO_BORDER; - uint32_t gslMinFilter = GSL_MIN_NEAREST; - uint32_t gslMagFilter = GSL_MAG_NEAREST; - bool unnorm = !(state & amd::Sampler::StateNormalizedCoordsMask); - - state &= ~amd::Sampler::StateNormalizedCoordsMask; - - // Program the sampler address mode - switch (state & amd::Sampler::StateAddressMask) { - case amd::Sampler::StateAddressRepeat: - gslAddress = GSL_REPEAT; - break; - case amd::Sampler::StateAddressClampToEdge: - gslAddress = GSL_CLAMP_TO_EDGE; - break; - case amd::Sampler::StateAddressMirroredRepeat: - gslAddress = GSL_MIRRORED_REPEAT; - break; - case amd::Sampler::StateAddressClamp: - case amd::Sampler::StateAddressNone: - default: - break; - } - state &= ~amd::Sampler::StateAddressMask; - - // Program texture filter mode - if (state == amd::Sampler::StateFilterLinear) { - gslMinFilter = GSL_MIN_LINEAR; - gslMagFilter = GSL_MAG_LINEAR; - } - - if (mipFilter == CL_FILTER_NEAREST) { - if (gslMinFilter == GSL_MIN_NEAREST) { - gslMinFilter = GSL_MIN_NEAREST_MIPMAP_NEAREST; - } else { - gslMinFilter = GSL_MIN_LINEAR_MIPMAP_NEAREST; - } - } else if (mipFilter == CL_FILTER_LINEAR) { - if (gslMinFilter == GSL_MIN_NEAREST) { - gslMinFilter = GSL_MIN_NEAREST_MIPMAP_LINEAR; - } else { - gslMinFilter = GSL_MIN_LINEAR_MIPMAP_LINEAR; - } - } - - fillSamplerHwState(unnorm, gslMinFilter, gslMagFilter, gslAddress, minLod, maxLod, hwState, - hwStateSize); -} - -void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const { - // for discrete gpu, we only reserve,no commit yet. - return amd::Os::reserveMemory(NULL, size, alignment, amd::Os::MEM_PROT_NONE); -} - -void Device::hostFree(void* ptr, size_t size) const { - // If we allocate the host memory, we need free, or we have to release - amd::Os::releaseMemory(ptr, size); -} - -void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_svm_mem_flags flags, - void* svmPtr) const { - alignment = std::max(alignment, static_cast(info_.memBaseAddrAlign_)); - - amd::Memory* mem = NULL; - if (NULL == svmPtr) { - if (isFineGrainedSystem()) { - return amd::Os::alignedMalloc(size, alignment); - } - - // create a hidden buffer, which will allocated on the device later - mem = new (context) amd::Buffer(context, flags, size, reinterpret_cast(1)); - if (mem == NULL) { - LogError("failed to create a svm mem object!"); - return NULL; - } - - if (!mem->create(NULL, false)) { - LogError("failed to create a svm hidden buffer!"); - mem->release(); - return NULL; - } - // if the device supports SVM FGS, return the committed CPU address directly. - gpu::Memory* gpuMem = getGpuMemory(mem); - - // add the information to context so that we can use it later. - amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem); - svmPtr = mem->getSvmPtr(); - } else { - // find the existing amd::mem object - mem = amd::MemObjMap::FindMemObj(svmPtr); - if (NULL == mem) { - return NULL; - } - // commit the CPU memory for FGS device. - if (isFineGrainedSystem()) { - mem->commitSvmMemory(); - } else { - gpu::Memory* gpuMem = getGpuMemory(mem); - } - svmPtr = mem->getSvmPtr(); - } - return svmPtr; -} - -void Device::svmFree(void* ptr) const { - if (isFineGrainedSystem()) { - amd::Os::alignedFree(ptr); - } else { - amd::Memory* svmMem = amd::MemObjMap::FindMemObj(ptr); - if (NULL != svmMem) { - svmMem->release(); - amd::MemObjMap::RemoveMemObj(ptr); - } - } -} - -Device::SrdManager::~SrdManager() { - for (uint i = 0; i < pool_.size(); ++i) { - pool_[i].buf_->unmap(NULL); - delete pool_[i].buf_; - delete pool_[i].flags_; - } -} - -bool Sampler::create(uint32_t oclSamplerState) { - hwSrd_ = dev_.srds().allocSrdSlot(&hwState_); - if (0 == hwSrd_) { - return false; - } - dev_.fillHwSampler(oclSamplerState, hwState_, HsaSamplerObjectSize); - return true; -} - -bool Sampler::create(const amd::Sampler& owner) { - hwSrd_ = dev_.srds().allocSrdSlot(&hwState_); - if (0 == hwSrd_) { - return false; - } - dev_.fillHwSampler(owner.state(), hwState_, HsaSamplerObjectSize, owner.mipFilter(), - owner.minLod(), owner.maxLod()); - return true; -} - -Sampler::~Sampler() { dev_.srds().freeSrdSlot(hwSrd_); } - -uint64_t Device::SrdManager::allocSrdSlot(address* cpuAddr) { - amd::ScopedLock lock(ml_); - // Check all buffers in the pool of chunks - for (uint i = 0; i < pool_.size(); ++i) { - const Chunk& ch = pool_[i]; - // Search for an empty slot - for (uint s = 0; s < numFlags_; ++s) { - uint mask = ch.flags_[s]; - // Check if there is an empty slot in this group - if (mask != 0) { - uint idx; - // Find the first empty index - for (idx = 0; (mask & 0x1) == 0; mask >>= 1, ++idx) - ; - // Mark the slot as busy - ch.flags_[s] &= ~(1 << idx); - // Calculate SRD offset in the buffer - uint offset = (s * MaskBits + idx) * srdSize_; - *cpuAddr = ch.buf_->data() + offset; - return ch.buf_->vmAddress() + offset; - } - } - } - // At this point the manager doesn't have empty slots - // and has to allocate a new chunk - Chunk chunk; - chunk.flags_ = new uint[numFlags_]; - if (chunk.flags_ == NULL) { - return 0; - } - chunk.buf_ = new Memory(dev_, bufSize_); - if (chunk.buf_ == NULL || !chunk.buf_->create(Resource::Remote) || - (NULL == chunk.buf_->map(NULL))) { - delete[] chunk.flags_; - delete chunk.buf_; - return 0; - } - // All slots in the chunk are in "free" state - memset(chunk.flags_, 0xff, numFlags_ * sizeof(uint)); - // Take the first one... - chunk.flags_[0] &= ~0x1; - pool_.push_back(chunk); - *cpuAddr = chunk.buf_->data(); - return chunk.buf_->vmAddress(); -} - -void Device::SrdManager::freeSrdSlot(uint64_t addr) { - amd::ScopedLock lock(ml_); - // Check all buffers in the pool of chunks - for (uint i = 0; i < pool_.size(); ++i) { - Chunk* ch = &pool_[i]; - // Find the offset - int64_t offs = static_cast(addr) - static_cast(ch->buf_->vmAddress()); - // Check if the offset inside the chunk buffer - if ((offs >= 0) && (offs < bufSize_)) { - // Find the index in the chunk - uint idx = offs / srdSize_; - uint s = idx / MaskBits; - // Free the slot - ch->flags_[s] |= 1 << (idx % MaskBits); - return; - } - } - assert(false && "Wrong slot address!"); -} - -void Device::SrdManager::fillResourceList(std::vector& memList) { - for (uint i = 0; i < pool_.size(); ++i) { - memList.push_back(pool_[i].buf_); - } -} - -int32_t Device::hwDebugManagerInit(amd::Context* context, uintptr_t messageStorage) { - int32_t status = hwDebugMgr_->registerDebugger(context, messageStorage); - - if (CL_SUCCESS != status) { - delete hwDebugMgr_; - hwDebugMgr_ = NULL; - } - - return status; -} - -bool Device::SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput) { - static const bool bValidate = true; - PerformAdapterInitialization(bValidate); - GSLClockModeInfo clockModeInfo = {}; - clockModeInfo.clockmode = static_cast(setClockModeInput.clock_mode); - bool result = gslSetClockMode(&clockModeInfo); - CloseInitializedAdapter(bValidate); - return result; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpudevice.hpp b/projects/clr/rocclr/device/gpu/gpudevice.hpp deleted file mode 100644 index 5f15d88140..0000000000 --- a/projects/clr/rocclr/device/gpu/gpudevice.hpp +++ /dev/null @@ -1,623 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPU_HPP_ -#define GPU_HPP_ - -#include "top.hpp" -#include "device/device.hpp" -#include "platform/command.hpp" -#include "platform/program.hpp" -#include "platform/perfctr.hpp" -#include "platform/threadtrace.hpp" -#include "platform/memory.hpp" -#include "utils/concurrent.hpp" -#include "thread/thread.hpp" -#include "thread/monitor.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "device/gpu/gpumemory.hpp" -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpusettings.hpp" -#include "device/gpu/gpuappprofile.hpp" - -#include - -#include "hsailctx.hpp" -#include "vaminterface.h" - -#if defined(__clang__) -#if __has_feature(address_sanitizer) -#include "device/devurilocator.hpp" -#endif -#endif - -/*! \addtogroup GPU - * @{ - */ - -//! GPU Device Implementation -namespace gpu { - -//! A nil device object -class NullDevice : public amd::Device { - protected: - static aclCompiler* compiler_; - static aclCompiler* hsaCompiler_; - - public: - aclCompiler* amdilCompiler() const { return compiler_; } - aclCompiler* hsaCompiler() const { return hsaCompiler_; } - aclCompiler* compiler() const { return hsaCompiler_; } - Compiler* binCompiler() const { return amdilCompiler(); } - - static bool init(void); - - //! Construct a new identifier - NullDevice(); - - //! Creates an offline device with the specified target - bool create(const char* calName, //!< GPU device name - const amd::Isa& isa, //!< GPU device isa - CALtarget target, //!< GPU device identifier - bool preferPal, //!< GPU prefer to use PAL if GPU_ENABLE_PAL=2 - bool doublePrecision, //!< Use double precision - bool openCL200 //!< Use OpenCL 2.0 - ); - - //! Instantiate a new virtual device - virtual device::VirtualDevice* createVirtualDevice(amd::CommandQueue* queue = NULL) { - return NULL; - } - - //! Create the device program. - virtual device::Program* createProgram(amd::Program& owner, amd::option::Options* options = NULL); - - //! Just returns NULL for the dummy device - virtual device::Memory* createMemory(amd::Memory& owner) const { return NULL; } - - //! Sampler object allocation - virtual bool createSampler(const amd::Sampler& owner, //!< abstraction layer sampler object - device::Sampler** sampler //!< device sampler object - ) const { - ShouldNotReachHere(); - return true; - } - - //! Just returns NULL for the dummy device - virtual device::Memory* createView( - amd::Memory& owner, //!< Owner memory object - const device::Memory& parent //!< Parent device memory object for the view - ) const { - return NULL; - } - - //! Signal object allocation - virtual device::Signal* createSignal() const { - return nullptr; - } - - //! Acquire external graphics API object in the host thread - //! Needed for OpenGL objects on CPU device - - virtual bool bindExternalDevice(uint flags, void* const pDevice[], void* pContext, - bool validateOnly) { - return true; - } - - virtual bool unbindExternalDevice(uint flags, void* const pDevice[], void* pContext, - bool validateOnly) { - return true; - } - - //! Releases non-blocking map target memory - virtual void freeMapTarget(amd::Memory& mem, void* target) {} - - CALtarget calTarget() const { return calTarget_; } - - CALMachineType calMachine() const { return calMachine_; } - - //! Empty implementation on Null device - virtual bool globalFreeMemory(size_t* freeMemory) const { return false; } - - //! Get GPU device settings - const gpu::Settings& settings() const { return reinterpret_cast(*settings_); } - virtual void* svmAlloc(amd::Context& context, size_t size, size_t alignment, - cl_svm_mem_flags flags, void* svmPtr) const { - return NULL; - } - virtual void svmFree(void* ptr) const { return; } - - virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput) { return true; } - -#if defined(__clang__) -#if __has_feature(address_sanitizer) - virtual device::UriLocator* createUriLocator() const { - return nullptr; - } -#endif -#endif - protected: - //! Answer the question: "Should HSAIL Program be created?", - //! based on the given options. - bool isHsailProgram(amd::option::Options* options = NULL); - - //! Fills OpenCL device info structure - void fillDeviceInfo(const CALdeviceattribs& calAttr, //!< CAL device attributes info - const gslMemInfo& memInfo, //!< GSL mem info - size_t maxTextureSize, //!< Maximum texture size supported in HW - uint numComputeRings, //!< Number of compute rings - uint numComputeRingsRT //!< Number of RT compute rings - ); - - CALtarget calTarget_; //!< GPU device identifier - CALMachineType calMachine_; //!< GPU machine identifier - const char* calName_; //!< GPU device name -}; - -//! Forward declarations -class Command; -class Device; -class GpuCommand; -class Heap; -class HeapBlock; -class Program; -class Kernel; -class Memory; -class Resource; -class VirtualDevice; -class PrintfDbg; -class ThreadTrace; - -#ifndef CL_FILTER_NONE -#define CL_FILTER_NONE 0x1142 -#endif - -class Sampler : public device::Sampler { - public: - //! Constructor - Sampler(const Device& dev) : dev_(dev) {} - - //! Default destructor for the device memory object - virtual ~Sampler(); - - //! Creates a device sampler from the OCL sampler state - bool create(uint32_t oclSamplerState //!< OCL sampler state - ); - - //! Creates a device sampler from the OCL sampler state - bool create(const amd::Sampler& owner //!< AMD sampler object - ); - - const void* hwState() const { return hwState_; } - - private: - //! Disable default copy constructor - Sampler& operator=(const Sampler&); - - //! Disable operator= - Sampler(const Sampler&); - - const Device& dev_; //!< Device object associated with the sampler - address hwState_; //!< GPU HW state (\todo legacy path) -}; - -//! A GPU device ordinal (physical GPU device) -class Device : public NullDevice, public CALGSLDevice { - public: - class Heap : public amd::EmbeddedObject { - public: - //! The size of a heap element in bytes - static constexpr size_t ElementSize = 4; - - //! The type of a heap element in bytes - static constexpr cmSurfFmt ElementType = CM_SURF_FMT_R32I; - - Heap() : resource_(NULL), baseAddress_(0) {} - - bool create(Device& device //!< GPU device object - ); - - //! Gets the GPU resource associated with the global heap - const Memory& resource() const { return *resource_; } - - //! Returns the base virtual address of the heap - uint64_t baseAddress() const { return baseAddress_; } - - protected: - Memory* resource_; //!< GPU resource referencing the heap memory - uint64_t baseAddress_; //!< Virtual heap base address - }; - - //! Locks any access to the virtual GPUs - class ScopedLockVgpus : public amd::StackObject { - public: - //! Default constructor - ScopedLockVgpus(const Device& dev); - - //! Destructor - ~ScopedLockVgpus(); - - private: - const Device& dev_; //! Device object - }; - - //! Interop emulation flags - enum InteropEmulationFlags { - D3D10Device = 0x00000001, - GLContext = 0x00000002, - }; - - class Engines : public amd::EmbeddedObject { - public: - //! Default constructor - Engines() : numComputeRings_(0), numComputeRingsRT_(0), numDmaEngines_(0) { - memset(desc_, 0xff, sizeof(desc_)); - } - - //! Creates engine descriptor for this class - void create(uint num, gslEngineDescriptor* desc, uint maxNumComputeRings); - - //! Gets engine type mask - uint getMask(gslEngineID id) const { return (1 << id); } - - //! Gets a descriptor for the requested engines - uint getRequested(uint engines, gslEngineDescriptor* desc) const; - - //! Returns the number of available compute rings - uint numComputeRings() const { return numComputeRings_; } - - //! Returns the number of available real time compute rings - uint numComputeRingsRT() const { return numComputeRingsRT_; } - - //! Returns the number of available DMA engines - uint numDMAEngines() const { return numDmaEngines_; } - - private: - uint numComputeRings_; - uint numComputeRingsRT_; - uint numDmaEngines_; - gslEngineDescriptor desc_[GSL_ENGINEID_MAX]; //!< Engine descriptor - }; - - //! Transfer buffers - class XferBuffers : public amd::HeapObject { - public: - static constexpr size_t MaxXferBufListSize = 8; - - //! Default constructor - XferBuffers(const Device& device, Resource::MemoryType type, size_t bufSize) - : type_(type), bufSize_(bufSize), acquiredCnt_(0), gpuDevice_(device) {} - - //! Default destructor - ~XferBuffers(); - - //! Creates the xfer buffers object - bool create(); - - //! Acquires an instance of the transfer buffers - Memory& acquire(); - - //! Releases transfer buffer - void release(VirtualGPU& gpu, //!< Virual GPU object used with the buffer - Memory& buffer //!< Transfer buffer for release - ); - - //! Returns the buffer's size for transfer - size_t bufSize() const { return bufSize_; } - - private: - //! Disable copy constructor - XferBuffers(const XferBuffers&); - - //! Disable assignment operator - XferBuffers& operator=(const XferBuffers&); - - //! Get device object - const Device& dev() const { return gpuDevice_; } - - Resource::MemoryType type_; //!< The buffer's type - size_t bufSize_; //!< Staged buffer size - std::list freeBuffers_; //!< The list of free buffers - std::atomic acquiredCnt_; //!< The total number of acquired buffers - amd::Monitor lock_; //!< Staged buffer acquire/release lock - const Device& gpuDevice_; //!< GPU device object - }; - - struct ScratchBuffer : public amd::HeapObject { - uint regNum_; //!< The number of used scratch registers - Memory* memObj_; //!< Memory objects for scratch buffers - uint64_t offset_; //!< Offset from the global scratch store - uint64_t size_; //!< Scratch buffer size on this queue - - //! Default constructor - ScratchBuffer() : regNum_(0), memObj_(NULL), offset_(0), size_(0) {} - - //! Default constructor - ~ScratchBuffer(); - - //! Destroys memory objects - void destroyMemory(); - }; - - - class SrdManager : public amd::HeapObject { - public: - SrdManager(const Device& dev, uint srdSize, uint bufSize) - : dev_(dev), - numFlags_(bufSize / (srdSize * MaskBits)), - srdSize_(srdSize), - bufSize_(bufSize) {} - ~SrdManager(); - - //! Allocates a new SRD slot for a resource - uint64_t allocSrdSlot(address* cpuAddr); - - //! Frees a SRD slot - void freeSrdSlot(uint64_t addr); - - // Fills the memory list for VidMM KMD - void fillResourceList(std::vector& memList); - - private: - //! Disable copy constructor - SrdManager(const SrdManager&); - - //! Disable assignment operator - SrdManager& operator=(const SrdManager&); - - struct Chunk { - Memory* buf_; - uint* flags_; - Chunk() : buf_(NULL), flags_(NULL) {} - }; - - static constexpr uint MaskBits = 32; - const Device& dev_; //!< GPU device for the chunk manager - amd::Monitor ml_; //!< Global lock for the SRD manager - std::vector pool_; //!< Pool of SRD buffers - uint numFlags_; //!< Total number of flags in array - uint srdSize_; //!< SRD size - uint bufSize_; //!< Buffer size that holds SRDs - }; - - //! Initialise the whole GPU device subsystem (CAL init, device enumeration, etc). - static bool init(); - - //! Shutdown the whole GPU device subsystem (CAL shutdown). - static void tearDown(); - - //! Construct a new physical GPU device - Device(); - - //! Initialise a device (i.e. all parts of the constructor that could - //! potentially fail) - bool create(CALuint ordinal, //!< GPU device ordinal index. Starts from 0 - CALuint numOfDevices //!< number of GPU devices in the system - ); - - //! Destructor for the physical GPU device - virtual ~Device(); - - //! Instantiate a new virtual device - device::VirtualDevice* createVirtualDevice(amd::CommandQueue* queue = NULL); - - //! Memory allocation - virtual device::Memory* createMemory(amd::Memory& owner //!< abstraction layer memory object - ) const; - - //! Sampler object allocation - virtual bool createSampler(const amd::Sampler& owner, //!< abstraction layer sampler object - device::Sampler** sampler //!< device sampler object - ) const; - - //! Allocates a view object from the device memory - virtual device::Memory* createView( - amd::Memory& owner, //!< Owner memory object - const device::Memory& parent //!< Parent device memory object for the view - ) const; - - //! Signal object allocation - virtual device::Signal* createSignal() const { - return nullptr; - } - - //! Create the device program. - virtual device::Program* createProgram(amd::Program& owner, amd::option::Options* options = NULL); - - //! Attempt to bind with external graphics API's device/context - virtual bool bindExternalDevice(uint flags, void* const pDevice[], void* pContext, - bool validateOnly); - - //! Attempt to unbind with external graphics API's device/context - virtual bool unbindExternalDevice(uint flags, void* const pDevice[], void* pContext, - bool validateOnly); - - //! Validates kernel before execution - virtual bool validateKernel(const amd::Kernel& kernel, //!< AMD kernel object - const device::VirtualDevice* vdev, - bool coop_groups = false); - - virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput); - - //! Retrieves information about free memory on a GPU device - virtual bool globalFreeMemory(size_t* freeMemory) const; - - //! Returns a GPU memory object from AMD memory object - gpu::Memory* getGpuMemory(amd::Memory* mem //!< Pointer to AMD memory object - ) const; - - //! Gets the GPU resource associated with the global heap - const Memory& globalMem() const { return heap_.resource(); } - - //! Gets the global heap object - const Heap& heap() const { return heap_; } - - //! Gets the memory object for the dummy page - amd::Memory* dummyPage() const { return dummyPage_; } - - amd::Monitor& lockAsyncOps() const { return *lockAsyncOps_; } - - //! Returns the lock object for the virtual gpus list - amd::Monitor* vgpusAccess() const { return vgpusAccess_; } - - //! Returns the number of virtual GPUs allocated on this device - uint numOfVgpus() const { return numOfVgpus_; } - uint numOfVgpus_; //!< The number of virtual GPUs (lock protected) - - typedef std::vector VirtualGPUs; - - //! Returns the list of all virtual GPUs running on this device - const VirtualGPUs& vgpus() const { return vgpus_; } - VirtualGPUs vgpus_; //!< The list of all running virtual gpus (lock protected) - - //! Scratch buffer allocation - gpu::Memory* createScratchBuffer(size_t size //!< Size of buffer - ) const; - - //! Returns transfer buffer object - XferBuffers& xferWrite() const { return *xferWrite_; } - - //! Returns transfer buffer object - XferBuffers& xferRead() const { return *xferRead_; } - - //! Finds an appropriate map target - amd::Memory* findMapTarget(size_t size) const; - - //! Adds a map target to the cache - bool addMapTarget(amd::Memory* memory) const; - - //! Returns resource cache object - ResourceCache& resourceCache() const { return *resourceCache_; } - - //! Returns engines object - const Engines& engines() const { return engines_; } - - //! Returns engines object - const device::BlitManager& xferMgr() const; - - VirtualGPU* xferQueue() const { return xferQueue_; } - - //! Retrieves the internal format from the OCL format - CalFormat getCalFormat(const amd::Image::Format& format //! OCL image format - ) const; - - //! Retrieves the OCL format from the internal image format - amd::Image::Format getOclFormat(const CalFormat& format //! Internal image format - ) const; - - const ScratchBuffer* scratch(uint idx) const { return scratch_[idx]; } - - //! Returns the global scratch buffer - Memory* globalScratchBuf() const { return globalScratchBuf_; }; - - //! Destroys scratch buffer memory - void destroyScratchBuffers(); - - //! Initialize heap resources if uninitialized - bool initializeHeapResources(); - - //! Set GSL sampler to the specified state - void fillHwSampler(uint32_t state, //!< Sampler's OpenCL state - void* hwState, //!< Sampler's HW state - uint32_t hwStateSize, //!< Size of sampler's HW state - uint32_t mipFilter = CL_FILTER_NONE, //!< Mip filter - float minLod = 0.f, //!< Min level of detail - float maxLod = CL_MAXFLOAT //!< Max level of detail - ) const; - - //! host memory alloc - virtual void* hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg = kNoAtomics) const; - - //! SVM allocation - virtual void* svmAlloc(amd::Context& context, size_t size, size_t alignment, - cl_svm_mem_flags flags, void* svmPtr) const; - - //! Free host SVM memory - void hostFree(void* ptr, size_t size) const; - - //! SVM free - virtual void svmFree(void* ptr) const; - - //! Returns SRD manger object - SrdManager& srds() const { return *srdManager_; } - - //! Initial the Hardware Debug Manager - int32_t hwDebugManagerInit(amd::Context* context, uintptr_t messageStorage); - -#if defined(__clang__) -#if __has_feature(address_sanitizer) - virtual device::UriLocator* createUriLocator() const { - return nullptr; - } -#endif -#endif - private: - //! Disable copy constructor - Device(const Device&); - - //! Disable assignment - Device& operator=(const Device&); - - //! Sends the stall command to all queues - bool stallQueues(); - - //! Buffer allocation - gpu::Memory* createBuffer(amd::Memory& owner, //!< Abstraction layer memory object - bool directAccess //!< Use direct host memory access - ) const; - - //! Image allocation - gpu::Memory* createImage(amd::Memory& owner, //!< Abstraction layer memory object - bool directAccess //!< Use direct host memory access - ) const; - - //! Allocates/reallocates the scratch buffer, according to the usage - bool allocScratch(uint regNum, //!< Number of the scratch registers - const VirtualGPU* vgpu //!< Virtual GPU for the allocation - ); - - Heap heap_; //!< GPU global heap - amd::Memory* dummyPage_; //!< A dummy page for NULL pointer - - amd::Monitor* lockAsyncOps_; //!< Lock to serialise all async ops on this device - amd::Monitor* lockAsyncOpsForInitHeap_; //!< Lock to serialise all async ops on initialization - //!heap operation - amd::Monitor* vgpusAccess_; //!< Lock to serialise virtual gpu list access - amd::Monitor* scratchAlloc_; //!< Lock to serialise scratch allocation - amd::Monitor* mapCacheOps_; //!< Lock to serialise cache for the map resources - - XferBuffers* xferRead_; //!< Transfer buffers read - XferBuffers* xferWrite_; //!< Transfer buffers write - - std::vector* mapCache_; //!< Map cache info structure - ResourceCache* resourceCache_; //!< Resource cache - Engines engines_; //!< Available engines on device - bool heapInitComplete_; //!< Keep track of initialization status of heap resources - VirtualGPU* xferQueue_; //!< Transfer queue - std::vector scratch_; //!< Scratch buffers for kernels - Memory* globalScratchBuf_; //!< Global scratch buffer - SrdManager* srdManager_; //!< SRD manager object - - static AppProfile appProfile_; //!< application profile -}; - -/*@}*/} // namespace gpu - -#endif /*GPU_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpukernel.cpp b/projects/clr/rocclr/device/gpu/gpukernel.cpp deleted file mode 100644 index 33d3da97f3..0000000000 --- a/projects/clr/rocclr/device/gpu/gpukernel.cpp +++ /dev/null @@ -1,3520 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/gpu/gpukernel.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpublit.hpp" -#include "device/gpu/gpuconstbuf.hpp" -#include "device/gpu/gpusched.hpp" -#include "platform/commandqueue.hpp" -#include "shader/ComputeProgramObject.h" -#include "utils/options.hpp" - -#include "hsailctx.hpp" -#include "SCCommon.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace gpu { - -const MetaDataConst ArgState[ArgStateTotal] = { - // Note: the order is important - // Name Type Properties - // Kernel description (special properties) - {"memory:compilerwrite", KernelArg::PrivateFixed, {0, 0, 0, 0, 0, 0, 0}}, - {"uniqueid:", KernelArg::NoType, {0, 0, 0, 0, 0, 0, 0}}, - {"memory:private:", KernelArg::PrivateSize, {0, 0, 0, 0, 0, 0, 0}}, - {"memory:local:", KernelArg::LocalSize, {0, 0, 0, 0, 0, 0, 0}}, - {"memory:hwprivate:", KernelArg::HwPrivateSize, {0, 0, 0, 0, 0, 0, 0}}, - {"memory:uavprivate:", KernelArg::HwPrivateSize, {0, 0, 0, 0, 0, 0, 0}}, - {"memory:hwlocal:", KernelArg::HwLocalSize, {0, 0, 0, 0, 0, 0, 0}}, - {"memory:64bitABI", KernelArg::ABI64Bit, {0, 0, 0, 0, 0, 0, 0}}, - {"limitgroupsize", KernelArg::Wavefront, {0, 0, 0, 0, 0, 0, 0}}, - {"function:", KernelArg::NoType, {1, 1, 0, 0, 0, 0, 0}}, - {"intrinsic:", KernelArg::NoType, {1, 0, 0, 0, 0, 0, 0}}, - {"error:", KernelArg::ErrorMessage, {0, 0, 0, 0, 0, 0, 0}}, - {"warning:", KernelArg::WarningMessage, {0, 0, 0, 0, 0, 0, 0}}, - {"printf_fmt:", KernelArg::PrintfFormatStr, {0, 0, 0, 0, 0, 0, 0}}, - {"version:", KernelArg::MetadataVersion, {0, 0, 0, 0, 0, 0, 0}}, - // Kernel basic types - {"pointer:", KernelArg::PointerGlobal, {1, 1, 1, 1, 1, 1, 0}}, - {"value:", KernelArg::Value, {1, 1, 1, 1, 1, 0, 0}}, - {"image:", KernelArg::Image, {1, 1, 1, 1, 1, 0, 0}}, - {"sampler:", KernelArg::Sampler, {0, 1, 0, 0, 0, 0, 0}}, - {"counter:", KernelArg::Counter, {1, 1, 0, 1, 1, 0, 0}}, - {"cws:", KernelArg::Grouping, {0, 0, 0, 0, 0, 0, 0}}, - {"lws:", KernelArg::WrkgrpSize, {0, 0, 0, 0, 0, 0, 0}}, - {"uavid:", KernelArg::UavId, {0, 0, 0, 0, 0, 0, 0}}, - {"reflection:", KernelArg::Reflection, {0, 0, 0, 0, 0, 0, 0}}, - {"constarg:", KernelArg::ConstArg, {0, 0, 0, 0, 0, 0, 0}}, - {"cbid:", KernelArg::ConstBufId, {0, 0, 0, 0, 0, 0, 0}}, - {"printfid:", KernelArg::PrintfBufId, {0, 0, 0, 0, 0, 0, 0}}, - {"wsh:", KernelArg::GroupingHint, {0, 0, 0, 0, 0, 0, 0}}, - {"vth:", KernelArg::VecTypeHint, {0, 0, 0, 0, 0, 0, 0}}, - {"WavesPerSimdHint:", KernelArg::WavesPerSimdHint, {0, 0, 0, 0, 0, 0, 0}}, -}; - -const DataTypeConst DataType[] = { - { - "i8:", KernelArg::Char, - }, - { - "i16:", KernelArg::Short, - }, - { - "i32:", KernelArg::Int, - }, - { - "i64:", KernelArg::Long, - }, - { - "u8:", KernelArg::UChar, - }, - { - "u16:", KernelArg::UShort, - }, - { - "u32:", KernelArg::UInt, - }, - { - "u64:", KernelArg::ULong, - }, - { - "float:", KernelArg::Float, - }, - { - "double:", KernelArg::Double, - }, - { - "struct:", KernelArg::Struct, - }, - { - "union:", KernelArg::Union, - }, - { - "1D:", KernelArg::Image1D, - }, - { - "2D:", KernelArg::Image2D, - }, - { - "3D:", KernelArg::Image3D, - }, - { - "1DB:", KernelArg::Image1DB, - }, - { - "1DA:", KernelArg::Image1DA, - }, - { - "2DA:", KernelArg::Image2DA, - }, - { - "opaque:", KernelArg::Opaque, - }, - { - "event:", KernelArg::Event, - }, - { - "sampler:", KernelArg::Sampler, - }, - { - "half:", KernelArg::Half, - }, -}; - -const uint DataTypeTotal = sizeof(DataType) / sizeof(DataTypeConst); - -struct BufDataConst { - const char* tagName_; //!< buffer's name - KernelArg::ArgumentType type_; //!< type of argument - struct { - uint number_ : 1; //!< buffer's number - uint alignment_ : 1; //!< buffer's alignment - uint attribute_ : 1; //!< buffer's read/write attribute - uint reserved : 29; //!< reserved - }; -}; - -static constexpr BufDataConst BufType[] = {{"g", KernelArg::PointerGlobal, {1, 0, 0, 0}}, - {"p", KernelArg::PointerPrivate, {1, 1, 1, 0}}, - {"l", KernelArg::PointerLocal, {1, 1, 1, 0}}, - {"uav", KernelArg::PointerGlobal, {1, 1, 1, 0}}, - {"c", KernelArg::PointerConst, {1, 1, 1, 0}}, - {"hl", KernelArg::PointerHwLocal, {1, 1, 1, 0}}, - {"hp", KernelArg::PointerHwPrivate, {1, 1, 1, 0}}, - {"hc", KernelArg::PointerHwConst, {1, 1, 1, 0}}}; -static constexpr uint BufTypeTotal = sizeof(BufType) / sizeof(BufDataConst); - -//! The mathlib constants for each kernel execution -static constexpr float MathLibConst[4] = {0.0f, 0.5f, 1.0f, 2.0f}; - -bool expect(const std::string& str, size_t* pos, const std::string& sym) { - bool result = true; - uint i; - - if (*pos == std::string::npos) { - return false; - } - - // Check if we have expected symbols - for (i = 0; i < sym.size(); ++i) { - char deb = str[*pos + i]; - if (deb != sym[i]) { - result = false; - break; - } - } - - if (result) *pos += i; - - return result; -} - -bool getword(const std::string& str, size_t* pos, std::string& sym) { - if (*pos == std::string::npos) { - return false; - } - - *pos = str.find_first_not_of(" \n\r", *pos); - size_t posEnd = str.find_first_of(": \n\r;", *pos); - size_t count = posEnd - *pos; - - if (count != 0) { - sym = str.substr(*pos, count); - } - sym[count] = 0; - *pos = posEnd + 1; - return true; -} - -bool getstring(const std::string& str, size_t* pos, std::string* out) { - if (*pos == std::string::npos) { - return false; - } - - *pos = str.find_first_not_of(" \n\r", *pos); - size_t posEnd = str.find_first_of(":\n\r;", *pos); - size_t count = posEnd - *pos; - - char* sym = new char[count + 1]; - if (count != 0) { - if (!str.copy(sym, count, *pos)) { - return false; - } - } - sym[count] = 0; - *out = sym; - delete[] sym; - *pos = posEnd + 1; - return true; -} - -bool getuint(const std::string& str, size_t* pos, uint* val) { - if (*pos == std::string::npos) { - return false; - } - - char sym[16]; - *pos = str.find_first_not_of(" \n\r", *pos); - size_t posEnd = str.find_first_of(": \n\r;)", *pos); - - if (!str.copy(sym, posEnd - *pos, *pos)) { - return false; - } - *val = 0; - for (size_t i = 0; i < (posEnd - *pos); ++i) { - *val = (*val * 10) + (sym[i] - 0x30); - } - *pos = posEnd + 1; - return true; -} - -bool getuintHex(const std::string& str, size_t* pos, uint* val) { - if (*pos == std::string::npos) { - return false; - } - - char sym[16]; - *pos = str.find_first_not_of(" \n\r", *pos); - size_t posEnd = str.find_first_of(": \n\r;)", *pos); - - if (!str.copy(sym, posEnd - *pos, *pos)) { - return false; - } - *val = 0; - for (size_t i = 0; i < (posEnd - *pos); ++i) { - if (sym[i] >= '0' && sym[i] <= 'F') { - *val = (*val * 16) + (sym[i] - '0'); - } else if (sym[i] >= 'a' && sym[i] <= 'f') { - *val = (*val * 16) + (sym[i] - 'a' + 10); - } else { - return false; - } - } - *pos = posEnd + 1; - return true; -} - -bool getuint64Hex(const std::string& str, size_t* pos, uint64_t* val) { - if (*pos == std::string::npos) { - return false; - } - - char sym[16]; - *pos = str.find_first_not_of(" \n\r", *pos); - size_t posEnd = str.find_first_of(": \n\r;)", *pos); - - if (!str.copy(sym, posEnd - *pos, *pos)) { - return false; - } - *val = 0; - for (size_t i = 0; i < (posEnd - *pos); ++i) { - if (sym[i] >= '0' && sym[i] <= 'F') { - *val = (*val * 16) + (sym[i] - '0'); - } else if (sym[i] >= 'a' && sym[i] <= 'f') { - *val = (*val * 16) + (sym[i] - 'a' + 10); - } else { - return false; - } - } - *pos = posEnd + 1; - return true; -} - -void intToStr(size_t value, char* str, size_t size) { - static const uint MaxDigits32bit = 10; - char result[MaxDigits32bit]; - uint idx = MaxDigits32bit; - - do { - idx--; - result[idx] = static_cast((value % 10) + '0'); - value /= 10; - } while ((value != 0) && (idx > 0)); - size_t len = MaxDigits32bit - idx; - size_t n = std::min(len, size - 1); - memcpy(str, &result[idx], n); - str[n] = '\0'; -} - -//! Default destructor -CalImageReference::~CalImageReference() { - // Free CAL image - free(image_); -} - -KernelArg::KernelArg() - : type_(KernelArg::NoType), - size_(0), - cbIdx_(0), - cbPos_(0), - index_(0), - alignment_(1), - dataType_(KernelArg::NoType) { - name_ = ""; - buf_ = ""; - memory_.value_ = 0; - typeQualifier_ = CL_KERNEL_ARG_TYPE_NONE; -} - -KernelArg::KernelArg(const KernelArg& data) { - // Fill the new object - *this = data; -} - -KernelArg& KernelArg::operator=(const KernelArg& data) { - // Fill the fields of the current object - name_ = data.name_; - typeName_ = data.typeName_; - typeQualifier_ = data.typeQualifier_; - type_ = data.type_; - size_ = data.size_; - cbIdx_ = data.cbIdx_; - cbPos_ = data.cbPos_; - buf_ = data.buf_; - index_ = data.index_; - alignment_ = data.alignment_; - dataType_ = data.dataType_; - memory_.value_ = data.memory_.value_; - return *this; -} - -bool KernelArg::isCbNeeded() const { - //! \note not a safe way - bool result = ((type_ > NoType) && (type_ < Sampler)) ? true : false; - if ((type_ == Sampler) && (location_ == 0)) { - // Sampler is defined outside the kernel - result = true; - } - return result; -} - -size_t KernelArg::size(bool gpuLayer) const { - switch (type_) { - case NoType: - return 0; - case PointerConst: - case PointerHwConst: - case PointerGlobal: - return (gpuLayer) ? sizeof(uint32_t) * size_ : sizeof(cl_mem); - case Image1D: - case Image2D: - case Image3D: - case Image1DB: - case Image1DA: - case Image2DA: - return (gpuLayer) ? sizeof(ImageConstants) : sizeof(cl_mem); - case Sampler: - return (gpuLayer) ? 2 * sizeof(uint32_t) : sizeof(cl_sampler); - case Counter: - return (gpuLayer) ? 0 : sizeof(cl_mem); - case PointerLocal: - case PointerHwLocal: - return (gpuLayer) ? sizeof(uint32_t) * size_ : sizeof(cl_mem); - case PointerPrivate: - case PointerHwPrivate: - return (gpuLayer) ? sizeof(uint32_t) * size_ : 0; - case Float: - return sizeof(float) * amd::nextPowerOfTwo(size_); - case Double: - return sizeof(double) * amd::nextPowerOfTwo(size_); - case Char: - case UChar: - return sizeof(cl_char) * amd::nextPowerOfTwo(size_); - case Short: - case UShort: - return sizeof(cl_short) * amd::nextPowerOfTwo(size_); - case Int: - case UInt: - return sizeof(uint32_t) * amd::nextPowerOfTwo(size_); - case Long: - case ULong: - return sizeof(uint64_t) * amd::nextPowerOfTwo(size_); - case Struct: - case Union: - return (gpuLayer) ? amd::alignUp(size_, 16) : size_; - default: - return 0; - } -} - -cl_kernel_arg_address_qualifier KernelArg::addressQualifier() const { - switch (type_) { - case PointerGlobal: - case Image1D: - case Image2D: - case Image3D: - case Image1DB: - case Image1DA: - case Image2DA: - return CL_KERNEL_ARG_ADDRESS_GLOBAL; - case PointerLocal: - case PointerHwLocal: - return CL_KERNEL_ARG_ADDRESS_LOCAL; - case PointerConst: - case PointerHwConst: - return CL_KERNEL_ARG_ADDRESS_CONSTANT; - default: - return CL_KERNEL_ARG_ADDRESS_PRIVATE; - } -} - -cl_kernel_arg_access_qualifier KernelArg::accessQualifier() const { - switch (type_) { - case Image1D: - case Image2D: - case Image3D: - case Image1DB: - case Image1DA: - case Image2DA: - if (memory_.readOnly_) { - return CL_KERNEL_ARG_ACCESS_READ_ONLY; - } else if (memory_.writeOnly_) { - return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; - } else if (memory_.readWrite_) { - return CL_KERNEL_ARG_ACCESS_READ_WRITE; - } - // Fall through ... - default: - return CL_KERNEL_ARG_ACCESS_NONE; - } -} - -//! temporary solution for the vectors handling in compiler -size_t KernelArg::specialVector() const { - if (size_ > VectorSizeLimit) { - switch (type_) { - case Char: - case UChar: - return sizeof(cl_char); - case Short: - case UShort: - return sizeof(cl_short); - default: - return 0; - } - } - return 0; -} -clk_value_type_t KernelArg::type() const { - switch (type_) { - case PointerGlobal: - case PointerLocal: - case PointerHwLocal: - case PointerConst: - case PointerHwConst: - case Image1D: - case Image2D: - case Image3D: - case Image1DB: - case Image1DA: - case Image2DA: - case Counter: - return T_POINTER; - case Float: - return T_FLOAT; - case Double: - return T_DOUBLE; - case Char: - case UChar: - return T_CHAR; - case Short: - case UShort: - return T_SHORT; - case Int: - return T_INT; - case UInt: - //! \note No UINT type - return T_INT; - case Long: - return T_LONG; - case ULong: - //! \note No ULONG type - return T_LONG; - case Struct: - case Union: - //! @todo What should we report? - return T_CHAR; - case Sampler: - return T_SAMPLER; - case PointerPrivate: - case PointerHwPrivate: - case NoType: - default: - return T_VOID; - } -} - -NullKernel::NullKernel(const std::string& name, const NullDevice& gpuNullDev, - const NullProgram& nullprog) - : device::Kernel(gpuNullDev, name, nullprog), - buildError_(CL_BUILD_PROGRAM_FAILURE), - gpuDev_(gpuNullDev), - calRef_(NULL), - internal_(false), - flags_(0), - cbSizes_(NULL), - numCb_(0), - rwAttributes_(false), - instructionCnt_(4) { - // UAV raw index will be detected - uavRaw_ = UavIdUndefined; - // CB index will be detected - cbId_ = UavIdUndefined; - // Printf index will be detected - printfId_ = UavIdUndefined; -} - -NullKernel::~NullKernel() { - uint idx; - - if (calRef_ == NULL) { - return; - } - calRef_->release(); - - // Destroy all kernel arguments - for (idx = 0; idx < arguments_.size(); ++idx) { - delete arguments_[idx]; - } - arguments_.clear(); - - // Destroy all sampler kernel arguments - for (idx = 0; idx < intSamplers_.size(); ++idx) { - delete intSamplers_[idx]; - } - intSamplers_.clear(); -} - - -static int scComponentToArrayIndex(E_SC_COMPONENT dstComp) { - switch (dstComp) { - case SC_COMPONENT_X: - return 0; - case SC_COMPONENT_Y: - return 1; - case SC_COMPONENT_Z: - return 2; - case SC_COMPONENT_W: - return 3; - } - - return 0; -} - -static void addLoopConst(const SC_HWSHADER* shader, AMUabiAddEncoding& encoding) { - uint count = shader->dep.NumIntrlIConstants; - encoding.litConstsCount = shader->dep.NumIntrlIConstants; - - // only suppport loop consts (int consts) - if (count) { - AMUabiLiteralConst* allocatedconsts = encoding.litConsts; - memset(allocatedconsts, 0, count * sizeof(AMUabiLiteralConst)); - uint usedConsts = 0; - for (uint i = 0; i < count; ++i) { - uint currentConst; - for (currentConst = 0; currentConst < usedConsts; ++currentConst) { - if (allocatedconsts[currentConst].addr == - HWSHADER_Get(shader, dep.IntrlIConstants)[i].uDstNumber) { - break; - } - } - if (currentConst == usedConsts) { - usedConsts++; - assert(usedConsts <= count); - } - allocatedconsts[currentConst].addr = HWSHADER_Get(shader, dep.IntrlIConstants)[i].uDstNumber; - allocatedconsts[currentConst].type = AMU_ABI_INT32; - allocatedconsts[currentConst].value.int32[scComponentToArrayIndex( - HWSHADER_Get(shader, dep.IntrlIConstants)[i].eDstComp)] = - HWSHADER_Get(shader, dep.IntrlIConstants)[i].iValue; - } - encoding.litConstsCount = usedConsts; - } -} - -bool NullKernel::create(const std::string& code, const std::string& metadata, - const void* binaryCode, size_t binarySize) { - std::auto_ptr uavRefCount(new uint[MaxUavArguments]); - if (NULL == uavRefCount.get()) { - return false; - } - - // Set all ref counts to 0 - memset(uavRefCount.get(), 0, sizeof(uavRefCount.get()[0]) * MaxUavArguments); - - // parse the metadata fields - if (!parseArguments(metadata, uavRefCount.get())) { - return false; - } - - CALimage calImage; -// Save source if DEBUG build -#if DEBUG - ilSource_ = code; -#endif // DEBUG - - amd::option::Options* options = nullProg().getCompilerOptions(); - internal_ = options->oVariables->clInternalKernel; - - if ((binaryCode == NULL) && (binarySize == 0) && !code.empty()) { - acl_error err; - aclTargetInfo info = amd::Hsail::GetTargetInfo(nullDev().settings().use64BitPtr_ ? "amdil64" : "amdil", - nullptr, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclGetTargetInfo failed"); - return false; - } - - aclBinaryOptions binOpts = {0}; - binOpts.struct_size = sizeof(binOpts); - binOpts.elfclass = info.arch_id == aclAMDIL64 ? ELFCLASS64 : ELFCLASS32; - binOpts.bitness = ELFDATA2LSB; - binOpts.alloc = &::malloc; - binOpts.dealloc = &::free; - - aclBinary* bin = amd::Hsail::BinaryInit(sizeof(aclBinary), &info, &binOpts, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclBinaryInit failed"); - return false; - } - - if (ACL_SUCCESS != - amd::Hsail::InsertSection(nullDev().amdilCompiler(), bin, code.data(), code.size(), aclSOURCE)) { - LogWarning("aclInsertSection failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - - amd::option::Options* Opts = (amd::option::Options*)bin->options; - - // Append an option so that we can selectively enable a SCOption on CZ - // whenever IOMMUv2 is enabled. - if (nullDev().settings().svmFineGrainSystem_) { - options->origOptionStr.append(" -sc-xnack-iommu"); - } - // temporary solution to synchronize buildNo between runtime and complib - // until we move runtime inside complib - Opts->setBuildNo(options->getBuildNo()); - - // pass kernel name to compiler - Opts->setCurrKernelName(name().c_str()); - - err = amd::Hsail::Compile(nullDev().amdilCompiler(), bin, options->origOptionStr.c_str(), ACL_TYPE_AMDIL_TEXT, - ACL_TYPE_ISA, NULL); - - buildLog_ += amd::Hsail::GetCompilerLog(nullDev().amdilCompiler()); - - if (err != ACL_SUCCESS) { - LogWarning("aclCompile failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - if (!options->oVariables->BinEXE) { - // Early exit if binary doesn't contain EXE - amd::Hsail::BinaryFini(bin); - return true; - } - size_t len; - const void* isa = amd::Hsail::ExtractSection(nullDev().amdilCompiler(), bin, &len, aclTEXT, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclExtractSection failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - - uint calImageSize; - if (!createMultiBinary(&calImageSize, reinterpret_cast(&calImage), isa)) { - LogWarning("initSrcEncoding failed"); - amd::Hsail::BinaryFini(bin); - return false; - } - - amd::Hsail::BinaryFini(bin); - } else if ((binaryCode != NULL) && (binarySize != 0)) { - uint size = 0; - if (!amuABIMultiBinaryGetSize(&size, const_cast(binaryCode)) || size > binarySize) { - buildLog_ += "Invalid binary image"; - LogError("amuABIMultiBinaryGetSize failed!"); - return false; - } - - calImage = static_cast(malloc(size)); - ::memcpy(calImage, binaryCode, size); - } else { - LogError("Incorrect initialization parameters!"); - return false; - } - - calRef_ = new CalImageReference(calImage); - if (calRef_ == NULL) { - LogError("Memory allocation failure!"); - // Free CAL image - free(calImage); - return false; - } - - CALfuncInfo calFuncInfo; - - // Get kernel compiled information - getFuncInfoFromImage(calImage, &calFuncInfo); - if (calFuncInfo.maxScratchRegsNeeded > 0) { - LogPrintfInfo( - "%s kernel has register spilling." - "Lower performance is expected.", - name().c_str()); - } - - workGroupInfo_.scratchRegs_ = calFuncInfo.maxScratchRegsNeeded; - workGroupInfo_.wavefrontPerSIMD_ = calFuncInfo.numWavefrontPerSIMD; - workGroupInfo_.wavefrontSize_ = calFuncInfo.wavefrontSize; - workGroupInfo_.availableGPRs_ = calFuncInfo.numGPRsAvailable; - workGroupInfo_.usedGPRs_ = calFuncInfo.numGPRsUsed; - workGroupInfo_.availableSGPRs_ = calFuncInfo.numSGPRsAvailable; - workGroupInfo_.usedSGPRs_ = calFuncInfo.numSGPRsUsed; - workGroupInfo_.availableVGPRs_ = calFuncInfo.numVGPRsAvailable; - workGroupInfo_.usedVGPRs_ = calFuncInfo.numVGPRsUsed; - workGroupInfo_.availableLDSSize_ = calFuncInfo.LDSSizeAvailable; - workGroupInfo_.usedLDSSize_ = calFuncInfo.LDSSizeUsed; - workGroupInfo_.availableStackSize_ = calFuncInfo.stackSizeAvailable; - workGroupInfo_.usedStackSize_ = calFuncInfo.stackSizeUsed; - - device::Kernel::parameters_t params; - if (!createSignature(params, params.size(), amd::KernelSignature::ABIVersion_0)) { - return false; - } - - return true; -} - -size_t NullKernel::getCalBinarySize() const { - CALuint imageSize; - if (!amuABIMultiBinaryGetSize(&imageSize, calImage())) { - LogError("Failed to get the image size!"); - return 0; - } - return static_cast(imageSize); -} - -bool NullKernel::getCalBinary(void* binary, size_t size) const { - uint calImageSize = 0; - if (!amuABIMultiBinaryGetSize(&calImageSize, calImage()) || size < calImageSize) { - LogError("CAL failed to save the kernel binary!"); - return false; - } - ::memcpy(binary, calImage(), calImageSize); - - return true; -} - -bool Kernel::create(const std::string& code, const std::string& metadata, const void* binaryCode, - size_t binarySize) { - setPreferredSizeMultiple(dev().getAttribs().wavefrontSize); - - if (!NullKernel::create(code, metadata, binaryCode, binarySize)) { - return false; - } - - // initialize constant buffer sizes - if (!initConstBuffers()) { - return false; - } - - // Initialize the kernel parameters - bool result = initParameters(); - - // Wave limiter needs to be initialized after kernel metadata is parsed - // Since it depends on it. - waveLimiter_.enable(dev().settings().ciPlus_); - - if (result) { - buildError_ = CL_SUCCESS; - } else { - result = false; - } - - return result; -} - -Kernel::Kernel(const std::string& name, const Device& gpuDev, const Program& prog, - const InitData* initData) - : NullKernel(name, gpuDev, prog) { - hwPrivateSize_ = 0; - if (NULL != initData) { - flags_ = initData->flags_; - hwPrivateSize_ = initData->hwPrivateSize_; - hwLocalSize_ = initData->hwLocalSize_; - } - // Workgroup info private memory size - workGroupInfo_.privateMemSize_ = hwPrivateSize_; - // Default wavesPerSimdHint_ - workGroupInfo_.wavesPerSimdHint_ = ~0U; -} - -Kernel::~Kernel() { - if (calRef_ == NULL) { - return; - } - - { - Device::ScopedLockVgpus lock(dev()); - - // Release all virtual image objects on all virtual GPUs - for (uint idx = 0; idx < dev().vgpus().size(); ++idx) { - dev().vgpus()[idx]->releaseKernel(calImage()); - } - } - - if (0 != numCb_) { - delete[] cbSizes_; - } -} - -const Device& Kernel::dev() const { return reinterpret_cast(gpuDev_); } - -const Program& Kernel::prog() const { return reinterpret_cast(prog_); } - -bool NullKernel::createMultiBinary(uint* imageSize, void** image, const void* isa) { - const SC_HWSHADER* shader = reinterpret_cast(isa); - - bool result = false; - AMUabiAddEncoding encoding; - memset(&encoding, 0, sizeof(AMUabiAddEncoding)); - - size_t allocSize = sizeof(uint) * MaxReadImage + sizeof(CALUavEntry) * MaxUavArguments + - sizeof(CALSamplerMapEntry) * MaxSamplers + sizeof(CALConstantBufferMask) * MaxConstBuffers + - sizeof(AMUabiLiteralConst) * shader->dep.NumIntrlIConstants; - char* tmpMem = new char[allocSize]; - if (tmpMem == NULL) { - LogError("Error allocating memory"); - return false; - } - - CalcPtr(encoding.inputs, tmpMem, 0, 0); - CalcPtr(encoding.uav, encoding.inputs, sizeof(uint), MaxReadImage); - CalcPtr(encoding.inputSamplerMaps, encoding.uav, sizeof(CALUavEntry), MaxUavArguments); - CalcPtr(encoding.constBuffers, encoding.inputSamplerMaps, sizeof(CALSamplerMapEntry), - MaxSamplers); - if (shader->dep.NumIntrlIConstants != 0) { - CalcPtr(encoding.litConsts, encoding.constBuffers, sizeof(CALConstantBufferMask), - MaxConstBuffers); - } - AMUabiMultiBinary amuBinary; - amuABIMultiBinaryCreate(&amuBinary); - - result = siCreateHwInfo(shader, encoding); - if (!result) { - delete[] tmpMem; - LogWarning("Error Creating program info"); - return false; - } - - addLoopConst(shader, encoding); - - unsigned int outputCount = 0, condOut = 0, earlyExit = 0, globalCount = 0, persistentCount = 0; - unsigned int symbolCount = 0; - CALOutputEntry* outputs = 0; - unsigned int* globalBuffers = 0; - unsigned int* persistentBuffers = 0; - AMUabiUserSymbol* symbols = 0; - - CALSamplerMapEntry* inputSamplers = encoding.inputSamplerMaps; - CALConstantBufferMask* constBuffers = encoding.constBuffers; - uint* inputResources = encoding.inputs; - CALUavEntry* uav = encoding.uav; - - uint inputSamplerCount = samplerSize(); - for (uint i = 0; i < inputSamplerCount; ++i) { - inputSamplers[i].resource = 0; - inputSamplers[i].sampler = sampler(i)->index_; - } - - uint constBufferCount = 2; - - constBuffers[0].index = 0; - constBuffers[1].index = 1; - - uint inputResourceCount = 0; - - uint uavCount = 0; - bool cbBound = false; - bool printfBound = false; - for (uint i = 0; i < arguments_.size(); ++i) { - const KernelArg* arg = argument(i); - switch (arg->type_) { - case KernelArg::PointerConst: - case KernelArg::PointerHwConst: - constBuffers[constBufferCount++].index = arg->index_; - break; - case KernelArg::PointerGlobal: - uav[uavCount].offset = arg->index_; - uav[uavCount].type = AMU_ABI_UAV_TYPE_TYPELESS; - uav[uavCount].dimension = AMU_ABI_DIM_BUFFER; - uav[uavCount].format = AMU_ABI_UAV_FORMAT_TYPELESS; - uavCount++; - break; - case KernelArg::ConstBufId: - if (!cbBound) { - uav[uavCount].offset = cbId_; - uav[uavCount].type = AMU_ABI_UAV_TYPE_RAW; - uav[uavCount].dimension = AMU_ABI_DIM_BUFFER; - uav[uavCount].format = AMU_ABI_UAV_FORMAT_TYPELESS; - uavCount++; - } - cbBound = true; - break; - case KernelArg::PrintfBufId: - if (!printfBound) { - uav[uavCount].offset = printfId_; - uav[uavCount].type = AMU_ABI_UAV_TYPE_RAW; - uav[uavCount].dimension = AMU_ABI_DIM_BUFFER; - uav[uavCount].format = AMU_ABI_UAV_FORMAT_TYPELESS; - uavCount++; - } - printfBound = true; - break; - case KernelArg::UavId: - if ((UavIdUndefined != uavRaw_) && !(flags() & PrintfOutput)) { - uav[uavCount].offset = arg->index_; - uav[uavCount].type = AMU_ABI_UAV_TYPE_TYPELESS; - uav[uavCount].dimension = AMU_ABI_DIM_BUFFER; - uav[uavCount].format = AMU_ABI_UAV_FORMAT_TYPELESS; - uavCount++; - } else { - if (UavIdUndefined != uavRaw_) { - uav[uavCount].offset = uavRaw_; - uav[uavCount].type = AMU_ABI_UAV_TYPE_RAW; - uav[uavCount].dimension = AMU_ABI_DIM_BUFFER; - uav[uavCount].format = AMU_ABI_UAV_FORMAT_TYPELESS; - uavCount++; - } - } - break; - case KernelArg::Sampler: - inputSamplers[inputSamplerCount].resource = 0; - inputSamplers[inputSamplerCount].sampler = arg->index_; - inputSamplerCount++; - break; - case KernelArg::Image1D: - case KernelArg::Image2D: - case KernelArg::Image3D: - case KernelArg::Image1DB: - case KernelArg::Image1DA: - case KernelArg::Image2DA: - if (arg->memory_.readOnly_) { - inputResources[inputResourceCount++] = arg->index_; - } else { - uav[uavCount].offset = arg->index_; - uav[uavCount].type = AMU_ABI_UAV_TYPE_TYPED; - uav[uavCount].dimension = AMU_ABI_DIM_2D; - uav[uavCount].format = AMU_ABI_UAV_FORMAT_TYPELESS; - uavCount++; - } - break; - default: - break; - } - } - - for (uint i = 0; i < nullProg().glbCb().size(); ++i) { - constBuffers[constBufferCount++].index = nullProg().glbCb()[i]; - } - - encoding.machine = nullDev().calMachine(); - encoding.type = ED_ATI_CAL_TYPE_COMPUTE; - encoding.inputCount = inputResourceCount; - encoding.outputCount = outputCount; - encoding.outputs = outputs; - encoding.condOut = condOut; - encoding.earlyExit = earlyExit; - encoding.globalBuffersCount = globalCount; - encoding.globalBuffers = globalBuffers; - encoding.persistentBuffersCount = persistentCount; - encoding.persistentBuffers = persistentBuffers; - encoding.constBuffersCount = constBufferCount; - encoding.inputSamplerMapCount = inputSamplerCount; - encoding.symbolsCount = symbolCount; - encoding.symbols = symbols; - encoding.uavCount = uavCount; - - amuABIMultiBinaryAddEncoding(amuBinary, &encoding); - - uint success = amuABIMultiBinaryPack(imageSize, image, amuBinary); - - amuABIMultiBinaryDestroy(amuBinary); - - delete[] tmpMem; - delete[] encoding.progInfos; - - return (success == 0) ? false : true; -} - -void Kernel::findLocalWorkSize(size_t workDim, const amd::NDRange& gblWorkSize, - amd::NDRange& lclWorkSize) const { - // Initialize the default workgoup info - // Check if the kernel has the compiled sizes - if (workGroupInfo()->compileSize_[0] == 0) { - // Find the default local workgroup size, if it wasn't specified - if (lclWorkSize[0] == 0) { - if ((dev().settings().overrideLclSet & (1 << (workDim - 1))) == 0) { - // Find threads per group - size_t thrPerGrp = workGroupInfo()->size_; - - // Check if kernel uses images - if ((flags() & ImageEnable) && - // and thread group is a multiple value of wavefronts - ((thrPerGrp % workGroupInfo()->wavefrontSize_) == 0) && - // and it's 2 or 3-dimensional workload - (workDim > 1) && ((gblWorkSize[0] % 16) == 0) && ((gblWorkSize[1] % 16) == 0)) { - // Use 8x8 workgroup size if kernel has image writes - if ((flags() & ImageWrite) || (thrPerGrp != nullDev().info().preferredWorkGroupSize_)) { - lclWorkSize[0] = 8; - lclWorkSize[1] = 8; - } else { - lclWorkSize[0] = 16; - lclWorkSize[1] = 16; - } - if (workDim == 3) { - lclWorkSize[2] = 1; - } - } else { - size_t tmp = thrPerGrp; - // Split the local workgroup into the most efficient way - for (uint d = 0; d < workDim; ++d) { - size_t div = tmp; - for (; (gblWorkSize[d] % div) != 0; div--) - ; - lclWorkSize[d] = div; - tmp /= div; - } - // Assuming DWORD access - const uint cacheLineMatch = dev().settings().cacheLineSize_ >> 2; - - // Check if we couldn't find optimal workload - if (((lclWorkSize.product() % workGroupInfo()->wavefrontSize_) != 0) || - // or size is too small for the cache line - (lclWorkSize[0] < cacheLineMatch)) { - size_t maxSize = 0; - size_t maxDim = 0; - for (uint d = 0; d < workDim; ++d) { - if (maxSize < gblWorkSize[d]) { - maxSize = gblWorkSize[d]; - maxDim = d; - } - } - // Use X dimension as high priority. Runtime will assume that - // X dimension is more important for the address calculation - if ((maxDim != 0) && (gblWorkSize[0] >= (cacheLineMatch / 2))) { - lclWorkSize[0] = cacheLineMatch; - thrPerGrp /= cacheLineMatch; - lclWorkSize[maxDim] = thrPerGrp; - for (uint d = 1; d < workDim; ++d) { - if (d != maxDim) { - lclWorkSize[d] = 1; - } - } - } else { - // Check if a local workgroup has the most optimal size - if (thrPerGrp > maxSize) { - thrPerGrp = maxSize; - } - lclWorkSize[maxDim] = thrPerGrp; - for (uint d = 0; d < workDim; ++d) { - if (d != maxDim) { - lclWorkSize[d] = 1; - } - } - } - } - } - } else { - // Use overrides when app doesn't provide workgroup dimensions - if (workDim == 1) { - lclWorkSize[0] = GPU_MAX_WORKGROUP_SIZE; - } else if (workDim == 2) { - lclWorkSize[0] = GPU_MAX_WORKGROUP_SIZE_2D_X; - lclWorkSize[1] = GPU_MAX_WORKGROUP_SIZE_2D_Y; - } else if (workDim == 3) { - lclWorkSize[0] = GPU_MAX_WORKGROUP_SIZE_3D_X; - lclWorkSize[1] = GPU_MAX_WORKGROUP_SIZE_3D_Y; - lclWorkSize[2] = GPU_MAX_WORKGROUP_SIZE_3D_Z; - } else { - assert(0 && "Invalid workDim!"); - } - } - } - } else { - for (uint d = 0; d < workDim; ++d) { - lclWorkSize[d] = workGroupInfo()->compileSize_[d]; - } - } -} - -void Kernel::setupProgramGrid(VirtualGPU& gpu, size_t workDim, const amd::NDRange& glbWorkOffset, - const amd::NDRange& gblWorkSize, amd::NDRange& lclWorkSize, - const amd::NDRange& groupOffset, const amd::NDRange& glbWorkOffsetOrg, - const amd::NDRange& glbWorkSizeOrg) const { - // ABI is always in CB0 - address cbBuf = gpu.cb(0)->sysMemCopy(); - uint* pGlobalSize = - reinterpret_cast(cbBuf + GlobalWorkitemOffset * ConstBuffer::VectorSize); - uint* pLocalSize = reinterpret_cast(cbBuf + LocalWorkitemOffset * ConstBuffer::VectorSize); - uint* pNumGroups = reinterpret_cast(cbBuf + GroupsOffset * ConstBuffer::VectorSize); - uint* pGlobalOffset = - reinterpret_cast(cbBuf + GlobalWorkOffsetOffset * ConstBuffer::VectorSize); - uint* pGroupOffset = - reinterpret_cast(cbBuf + GroupWorkOffsetOffset * ConstBuffer::VectorSize); - uint32_t* debugInfo = reinterpret_cast(cbBuf + DebugOffset * ConstBuffer::VectorSize); - uint* pNDRangeGlobalOffset = - reinterpret_cast(cbBuf + NDRangeGlobalWorkOffsetOffset * ConstBuffer::VectorSize); - - // Check for 64-bit metadata - uint glbABIShift = (abi64Bit()) ? 1 : 0; - - VirtualGPU::CalVirtualDesc* progGrid = &gpu.cal_; - - // Finds local workgroup size - findLocalWorkSize(workDim, gblWorkSize, lclWorkSize); - - // Initialize the execution grid block and size/offset - pGlobalSize[0] = pGlobalSize[1] = pGlobalSize[2] = 1; - pGlobalSize[3] = static_cast(workDim); - - pLocalSize[0] = pLocalSize[1] = pLocalSize[2] = 1; - pLocalSize[3] = 0; - - pNumGroups[0] = pNumGroups[1] = pNumGroups[2] = 1; - pNumGroups[3] = 0; - - pGlobalOffset[2] = pGlobalOffset[1] = pGlobalOffset[0] = 0; - pGroupOffset[2] = pGroupOffset[1] = pGroupOffset[0] = 0; - - progGrid->gridBlock.width = progGrid->gridBlock.height = progGrid->gridBlock.depth = 1; - - progGrid->gridSize.width = progGrid->gridSize.height = progGrid->gridSize.depth = 1; - - progGrid->partialGridBlock.width = progGrid->partialGridBlock.height = - progGrid->partialGridBlock.depth = 1; - - bool partialGrid = false; - - // Fill the right values, based on the application request - switch (workDim) { - case 3: - pLocalSize[2] = progGrid->gridBlock.depth = static_cast(lclWorkSize[2]); - - pGlobalSize[2] = static_cast(glbWorkSizeOrg[2]); - progGrid->gridSize.depth = static_cast(gblWorkSize[2]); - progGrid->gridSize.depth /= progGrid->gridBlock.depth; - pNumGroups[2] = pGlobalSize[2] / progGrid->gridBlock.depth; - - pGlobalOffset[2] = glbWorkOffset[2]; - pGroupOffset[2] = groupOffset[2]; - pNDRangeGlobalOffset[2 + glbABIShift] = glbWorkOffsetOrg[2]; - - // Check if partial workgroup dispatch is required - progGrid->partialGridBlock.depth = gblWorkSize[2] % lclWorkSize[2]; - if (progGrid->partialGridBlock.depth != 0) { - partialGrid = true; - // Increment the number of groups - progGrid->gridSize.depth++; - pNumGroups[2]++; - } else { - progGrid->partialGridBlock.depth = lclWorkSize[2]; - } - // Fall through to fill 2D and 1D dimensions... - case 2: - pLocalSize[1] = progGrid->gridBlock.height = static_cast(lclWorkSize[1]); - - pGlobalSize[1] = static_cast(glbWorkSizeOrg[1]); - progGrid->gridSize.height = static_cast(gblWorkSize[1]); - progGrid->gridSize.height /= progGrid->gridBlock.height; - pNumGroups[1] = pGlobalSize[1] / progGrid->gridBlock.height; - - pGlobalOffset[1] = glbWorkOffset[1]; - pGroupOffset[1] = groupOffset[1]; - pNDRangeGlobalOffset[1 + glbABIShift] = glbWorkOffsetOrg[1]; - - // Check if partial workgroup dispatch is required - progGrid->partialGridBlock.height = gblWorkSize[1] % lclWorkSize[1]; - if (progGrid->partialGridBlock.height != 0) { - partialGrid = true; - // Increment the number of groups - progGrid->gridSize.height++; - pNumGroups[1]++; - } else { - progGrid->partialGridBlock.height = lclWorkSize[1]; - } - // Fall through to fill 1D dimension... - case 1: - pLocalSize[0] = progGrid->gridBlock.width = static_cast(lclWorkSize[0]); - - pGlobalSize[0] = static_cast(glbWorkSizeOrg[0]); - progGrid->gridSize.width = static_cast(gblWorkSize[0]); - progGrid->gridSize.width /= progGrid->gridBlock.width; - pNumGroups[0] = pGlobalSize[0] / progGrid->gridBlock.width; - - pGlobalOffset[0] = glbWorkOffset[0]; - pGroupOffset[0] = groupOffset[0]; - pNDRangeGlobalOffset[0 + glbABIShift] = glbWorkOffsetOrg[0]; - - // Check if partial workgroup dispatch is required - progGrid->partialGridBlock.width = gblWorkSize[0] % lclWorkSize[0]; - if (progGrid->partialGridBlock.width != 0) { - partialGrid = true; - // Increment the number of groups - progGrid->gridSize.width++; - pNumGroups[0]++; - } else { - progGrid->partialGridBlock.width = lclWorkSize[0]; - } - break; - default: - LogWarning("Wrong dimensions. Force to 1x1x1!"); - break; - } - - if (!partialGrid) { - progGrid->partialGridBlock.width = progGrid->partialGridBlock.height = - progGrid->partialGridBlock.depth = 0; - } - - // Calculate the total number of workitems and workgroups - pGlobalOffset[3] = pGroupOffset[3] = 1; - for (uint i = 0; i < workDim; ++i) { - pGlobalOffset[3] *= pGlobalOffset[i]; - pGroupOffset[3] *= pGroupOffset[i]; - } - - // Setup debug output buffer (if printf is active) - if (flags() & PrintfOutput) { - if (abi64Bit()) { - // Setup the debug info in constant buffer - reinterpret_cast(debugInfo)[1] = gpu.printfDbg().bufOffset(); - // Size in DWORDs - debugInfo[4] = static_cast(gpu.printfDbg().wiDbgSize()); - debugInfo[4] /= sizeof(uint32_t); - } else { - // Setup the debug info in constant buffer - debugInfo[1] = static_cast(gpu.printfDbg().bufOffset()); - // Size in DWORDs - debugInfo[2] = static_cast(gpu.printfDbg().wiDbgSize()); - debugInfo[2] /= sizeof(uint32_t); - } - } -} - -bool Kernel::initParameters() { - size_t offset = 0; - device::Kernel::parameters_t params; - amd::KernelParameterDescriptor desc; - - for (uint i = 0; i < arguments_.size(); ++i) { - const KernelArg* arg = argument(i); - - // Initialize the arguments for the abstraction layer - if (arg->isCbNeeded()) { - desc.name_ = arg->name_.data(); - desc.type_ = arg->type(); - desc.size_ = arg->size(false); - desc.addressQualifier_ = arg->addressQualifier(); - desc.accessQualifier_ = arg->accessQualifier(); - desc.typeName_ = arg->typeName(); - desc.typeQualifier_ = arg->typeQualifier(); - - // Make offset alignment to match CPU metadata, since - // in multidevice config abstraction layer has a single signature - // and CPU sends the paramaters as they are allocated in memory - size_t size = desc.size_; - if (size == 0) { - // Local memory for CPU - size = sizeof(cl_mem); - } - offset = amd::alignUp(offset, std::min(size, size_t(16))); - desc.offset_ = offset; - offset += amd::alignUp(size, sizeof(uint32_t)); - params.push_back(desc); - } - } - - // Report the allocated local memory size (emulated and hw) - if (hwLocalSize_ != 0) { - CondLog((dev().info().localMemSize_ < hwLocalSize_), - "Requested local size is bigger than reported"); - workGroupInfo_.localMemSize_ = hwLocalSize_; - } - - if (!createSignature(params, params.size(), amd::KernelSignature::ABIVersion_0)) { - return false; - } - - return true; -} - -bool Kernel::bindGlobalHwCb(VirtualGPU& gpu, VirtualGPU::GslKernelDesc* desc) const { - bool result = true; - - // Bind HW constant buffers used for the global data store - const Program::HwConstBuffers& gds = prog().glbHwCb(); - for (const auto& it : gds) { - uint idx = it.first; - result = bindResource(gpu, *(it.second), idx, ConstantBuffer, idx); - } - - return result; -} - -bool Kernel::bindConstantBuffers(VirtualGPU& gpu) const { - bool result = true; - - assert((numCb_ <= MaxConstBuffersArguments) && "Runtime doesn't support more CBs for arguments!"); - - // Upload the parameters to HW and bind all constant buffers - for (uint i = 0; i < numCb_; i++) { - ConstBuffer* cb = gpu.constBufs_[i]; - result &= cb->uploadDataToHw(cbSizes_[i]) && - bindResource(gpu, *cb, i, ConstantBuffer, i, cb->wrtOffset()); - } - - return result; -} - -void Kernel::processMemObjects(VirtualGPU& gpu, const amd::Kernel& kernel, const_address params, - bool nativeMem) const { - // Mark the tracker with a new kernel, - // so we can avoid checks of the aliased objects - gpu.memoryDependency().newKernel(); - - // Check all parameters for the current kernel - const amd::KernelSignature& signature = kernel.signature(); - amd::Memory* const* memories = - reinterpret_cast(params + kernel.parameters().memoryObjOffset()); - - for (size_t i = 0; i < signature.numParameters(); ++i) { - const amd::KernelParameterDescriptor& desc = signature.at(i); - const KernelArg* arg = argument(i); - Memory* memory = NULL; - - // Find if current argument is a buffer - if ((desc.type_ == T_POINTER) && (arg->type_ != KernelArg::PointerLocal) && - (arg->type_ != KernelArg::PointerHwLocal)) { - uint32_t index = desc.info_.arrayIndex_; - if (nativeMem) { - memory = reinterpret_cast(memories)[index]; - } else if (*reinterpret_cast(params + desc.offset_) != NULL) { - memory = dev().getGpuMemory(memories[index]); - // Synchronize data with other memory instances if necessary - memory->syncCacheFromHost(gpu); - } - - if (memory != NULL) { - // Validate memory for a dependency in the queue - gpu.memoryDependency().validate(gpu, memory, arg->memory_.readOnly_); - } - } - } -} - -bool Kernel::loadParameters(VirtualGPU& gpu, const amd::Kernel& kernel, const_address params, - bool nativeMem) const { - bool result = true; - uint i; - - // Initialize local private ranges - if (!initLocalPrivateRanges(gpu)) { - return false; - } - - if ((UavIdUndefined != uavRaw_) && (!(flags() & PrintfOutput) || (printfId_ != UavIdUndefined))) { - Memory* gpuMemory = dev().getGpuMemory(dev().dummyPage()); - // Bind a buffer for a dummy read - result = bindResource(gpu, *gpuMemory, 0, ArgumentUavID, uavRaw_); - } - - // Find all parameters for the current kernel - const amd::KernelSignature& signature = kernel.signature(); - for (i = 0; i != signature.numParameters(); ++i) { - const amd::KernelParameterDescriptor& desc = signature.at(i); - // Set current argument - if (!setArgument(gpu, kernel, i, params, desc, nativeMem)) { - result = false; - break; - } - } - - if (result) { - // Update the ring ranges and math constant - setLocalPrivateRanges(gpu); - - result = bindConstantBuffers(gpu); - - if (flags() & PrivateFixed) { - result &= bindResource(gpu, dev().globalMem(), 0, GlobalBuffer, uavRaw_); - } - - // Setup debug output buffer (if printf is active) - if (flags() & PrintfOutput) { - gpu.addVmMemory(gpu.printfDbg().dbgBuffer()); - } - } - - return result; -} - -bool Kernel::run(VirtualGPU& gpu, GpuEvent* calEvent, bool lastRun, bool lastDoppCmd, - bool pfpaDoppCmd) const { - const VirtualGPU::CalVirtualDesc* dispatch = gpu.cal(); - - auto compProg = static_cast(gpu.gslKernelDesc()->func_); - compProg->setWavesPerSH(waveLimiter_.getWavesPerSH(&gpu)); - - gpu.eventBegin(MainEngine); - gpu.rs()->Dispatch(gpu.cs(), &dispatch->gridBlock, &dispatch->partialGridBlock, - &dispatch->gridSize, dispatch->localSize, gpu.vmMems(), dispatch->memCount_, - lastDoppCmd, pfpaDoppCmd); - - gpu.flushCUCaches(); - - gpu.eventEnd(MainEngine, *calEvent); - - // Unbind all resources - unbindResources(gpu, *calEvent, lastRun); - - return true; -} - -static size_t counter = 0; -void Kernel::debug(VirtualGPU& gpu) const { - std::fstream stubWrite; - address src = NULL; - - std::cerr << "--- " << name_ << " ---" << std::endl; - for (uint i = 0; i < arguments_.size(); ++i) { - const KernelArg* arg = argument(i); - const Memory* gpuMem = gpu.slots_[i].memory_; - std::stringstream fileName; - bool bufferObj = - ((arg->type_ == KernelArg::PointerGlobal) || (arg->type_ == KernelArg::PointerConst) || - (arg->type_ == KernelArg::PointerHwConst)); - - if ((src != NULL) && arg->isCbNeeded() && bufferObj) { - address memory = gpu.cb(arg->cbIdx_)->sysMemCopy(); - std::cerr.setf(std::ios::hex); - uint* location = - reinterpret_cast(src + *reinterpret_cast(memory + arg->cbPos_)); - std::cerr << " > " << arg->name_ << ": 0x" << location << std::endl; - - // Dump the data - fileName << counter << "_kernel_" << name() << "_" << arg->name_ << "_" << location << ".bin"; - stubWrite.open(fileName.str().c_str(), (std::fstream::out | std::fstream::binary)); - - // Write data to a file - if (stubWrite.is_open()) { - stubWrite.write(reinterpret_cast(location), gpuMem->size()); - stubWrite.close(); - } - } - if (((arg->type_ >= KernelArg::Image1D) && (arg->type_ <= KernelArg::Image3D)) || - ((src == NULL) && bufferObj)) { - //@todo Replace the current map - Memory* resource = const_cast(gpu.slots_[i].memory_); - void* memory = resource->map(&gpu); - uint* location = reinterpret_cast(memory); - std::cerr << " > " << arg->name_ << (bufferObj ? ": buffer" : ": image") << std::endl; - // Dump the data - fileName << counter << "_kernel_" << name() << "_" << arg->name_ << "_" << location << ".bin"; - stubWrite.open(fileName.str().c_str(), (std::fstream::out | std::fstream::binary)); - - // Write data to a file - if (stubWrite.is_open()) { - stubWrite.write(reinterpret_cast(location), gpuMem->size()); - stubWrite.close(); - } - resource->unmap(&gpu); - } - } - - for (uint i = 0; i < gpu.constBufs_.size(); ++i) { - std::stringstream fileName; - fileName << counter++ << "_kernel_" << name() << "_const" << i << ".bin"; - stubWrite.open(fileName.str().c_str(), (std::fstream::out | std::fstream::binary)); - if (stubWrite.is_open()) { - address memory = reinterpret_cast
(gpu.constBufs_[i]->map(&gpu, Resource::ReadOnly)); - // Check if we have OpenCL program - stubWrite.write(reinterpret_cast(memory + gpu.cb(i)->wrtOffset()), - gpu.cb(i)->lastWrtSize()); - gpu.constBufs_[i]->unmap(&gpu); - stubWrite.close(); - } - } - const Program::HwConstBuffers& gds = prog().glbHwCb(); - for (const auto& it : gds) { - uint idx = it.first; - std::stringstream fileName; - fileName << counter++ << "_kernel_" << name() << "_const" << idx << ".bin"; - stubWrite.open(fileName.str().c_str(), (std::fstream::out | std::fstream::binary)); - if (stubWrite.is_open()) { - address memory = reinterpret_cast
(it.second->map(&gpu, Resource::ReadOnly)); - // Check if we have OpenCL program - stubWrite.write(reinterpret_cast(memory), it.second->size()); - it.second->unmap(&gpu); - stubWrite.close(); - } - } -} - -bool Kernel::initConstBuffers() { - bool result = true; - size_t i; - - assert((numCb_ != 0) && "We have 0 constant buffers!"); - - // Allocate an array for CB sizes - cbSizes_ = new size_t[numCb_]; - if (cbSizes_ == NULL) { - return false; - } - memset(cbSizes_, 0, sizeof(size_t) * numCb_); - - // CB0 is reserved for ABI data - cbSizes_[0] = TotalABIVectors * ConstBuffer::VectorSize; - - // Find sizes of all constant buffers - for (i = 0; i < arguments_.size(); ++i) { - const KernelArg* arg = argument(i); - size_t size = arg->cbPos_ + arg->size(true); - size_t specVec = arg->specialVector(); - if (specVec != 0) { - size = arg->cbPos_ + (arg->size_ / KernelArg::VectorSizeLimit) * ConstBuffer::VectorSize; - } - // Do we need a CB? - if (arg->isCbNeeded() && (cbSizes_[arg->cbIdx_] < size)) { - cbSizes_[arg->cbIdx_] = size; - } - } - - return result; -} - -bool Kernel::setInternalSamplers(VirtualGPU& gpu) const { - for (uint i = 0; i < samplerSize(); ++i) { - const KernelArg* arg = sampler(i); - uint state = arg->cbPos_; - uint idx = arg->index_; - - if (gpu.cal()->samplersState_[idx] != state) { - setSampler(gpu, state, idx); - gpu.cal_.samplersState_[idx] = state; - } - } - - return true; -} - -bool Kernel::setArgument(VirtualGPU& gpu, const amd::Kernel& kernel, - uint idx, const_address params, - const amd::KernelParameterDescriptor& desc, - bool nativeMem) const { - size_t size = desc.size_; - const void* param = params + desc.offset_; - bool result = true; - const KernelArg* arg; - address memory; - size_t argSize; - static const bool waitOnBusyEngine = true; - - assert((idx < arguments_.size()) && "Param index is out of range!"); - - arg = argument(idx); - assert((arg->cbIdx_ == 1) && "Runtime supports CB1 only for the arguments buffer!"); - memory = gpu.cb(1)->sysMemCopy(); - argSize = arg->size(true); - - // Bind the global heap for emulation mode - switch (arg->type_) { - case KernelArg::PointerLocal: - case KernelArg::PointerPrivate: - if (!bindResource(gpu, dev().globalMem(), 0, GlobalBuffer, uavRaw_)) { - return false; - } - // Fall through ... - default: - break; - } - - switch (arg->type_) { - case KernelArg::PointerConst: - case KernelArg::PointerHwConst: - case KernelArg::PointerGlobal: { - gpu::Memory* gpuMem = NULL; - amd::Memory* const* memories = - reinterpret_cast(params + kernel.parameters().memoryObjOffset()); - uint32_t index = desc.info_.arrayIndex_; - if (nativeMem) { - gpuMem = reinterpret_cast(memories[index]); - } else if (memories[index] != nullptr) { - gpuMem = dev().getGpuMemory(memories[index]); - } - bool forceZeroOffset = false; - - if (gpuMem == NULL) { - forceZeroOffset = true; - gpuMem = dev().getGpuMemory(dev().dummyPage()); - } - uint64_t offset = gpuMem->pinOffset(); - - // Make sure the passed argument is a buffer object - if (!gpuMem->cal()->buffer_) { - LogError("The kernel buffer argument isn't a buffer object!"); - return false; - } - - if (arg->type_ == KernelArg::PointerHwConst) { - // Bind current memory object with the kernel - if (!bindResource(gpu, *gpuMem, idx, ArgumentConstBuffer, arg->index_)) { - return false; - } - assert((offset == 0) && "No offset for HW CB"); - // Add a fake offset to make sure (ptr != NULL) is TRUE - offset = 1; - } else { - ResourceType type = ArgumentHeapBuffer; - - // Check if kernel expects UAV binding - if (arg->memory_.uavBuf_) { - type = ArgumentBuffer; - } else { - // Bind global buffer to UAV this buffer is bound to - if (!bindResource(gpu, dev().globalMem(), 0, GlobalBuffer, uavRaw_)) { - return false; - } - } - - // Bind current memory object with the kernel - // Note: it's a fake binding, if the buffer is part of - // the global heap - if (!bindResource(gpu, *gpuMem, idx, type, arg->index_)) { - return false; - } - - // Update offset only if we bind HeapBuffer or - // it's global address space in UAV setup on SI+ - offset += gpuMem->hbOffset(); - if (!forceZeroOffset) { - assert((offset != 0) && "Offset 0 with a real allocation!"); - } - gpu.addVmMemory(gpuMem); - } - - // Wait for resource if it was used on an inactive engine - //! \note syncCache may call DRM transfer - gpuMem->wait(gpu, waitOnBusyEngine); - - if (forceZeroOffset) { - offset = 0; - } - - // Copy memory offset into the constant buffer - if (abi64Bit()) { - *(reinterpret_cast(memory + arg->cbPos_)) = offset; - } else { - *(reinterpret_cast(memory + arg->cbPos_)) = static_cast(offset); - } - } break; - case KernelArg::Image1D: - case KernelArg::Image2D: - case KernelArg::Image3D: - case KernelArg::Image1DB: - case KernelArg::Image1DA: - case KernelArg::Image2DA: { - gpu::Memory* gpuMem = NULL; - amd::Memory* const* memories = - reinterpret_cast(params + kernel.parameters().memoryObjOffset()); - uint32_t index = desc.info_.arrayIndex_; - if (nativeMem) { - gpuMem = reinterpret_cast(memories[index]); - } else if (memories[index] != nullptr) { - gpuMem = dev().getGpuMemory(memories[index]); - } - - if (gpuMem == NULL) { - return false; - } - // Make sure the passed argument is an image object - if (gpuMem->cal()->buffer_) { - LogError("The kernel image argument isn't an image object!"); - return false; - } - - ResourceType resType = arg->memory_.readOnly_ ? ArgumentImageRead : ArgumentImageWrite; - - // Bind current memory object with the shader. - if (!bindResource(gpu, *gpuMem, idx, resType, arg->index_)) { - return false; - } - - // Wait for resource if it was used on an inactive engine - //! \note syncCache may call DRM transfer - gpuMem->wait(gpu, waitOnBusyEngine); - - // Copy image constants into the constant buffer - if (gpuMem->owner() != NULL) { - copyImageConstants(gpuMem->owner()->asImage(), - reinterpret_cast(memory + arg->cbPos_)); - } - - // Handle DOPP texture resource - gslMemObject gslMem = gpuMem->gslResource(); - if (gslMem->getAttribs().isDOPPDesktopTexture) { - gpu.addVmMemory(gpuMem); - } - } break; - case KernelArg::Sampler: { - uint32_t index = desc.info_.arrayIndex_; - const amd::Sampler* amdSampler = reinterpret_cast(params + - kernel.parameters().samplerObjOffset())[index]; - uint idx = arg->index_; - uint32_t state = amdSampler->state(); - - if (state != gpu.cal()->samplersState_[idx]) { - setSampler(gpu, state, idx); - gpu.cal_.samplersState_[idx] = state; - } - - // Copy sampler state into the constant buffer - *(reinterpret_cast(memory + arg->cbPos_)) = state; - } break; - case KernelArg::Counter: { - gpu::Memory* gpuMem = NULL; - if (nativeMem) { - gpuMem = *reinterpret_cast(param); - } else if (*reinterpret_cast(param) != NULL) { - gpuMem = dev().getGpuMemory(*reinterpret_cast(param)); - } - - // Wait for resource if it was used on an inactive engine - //! \note syncCache may call DRM transfer - gpuMem->wait(gpu, waitOnBusyEngine); - - // Bind current memory object with the shader. - if (!bindResource(gpu, *gpuMem, idx, ArgumentCounter, idx)) { - return false; - } - } break; - case KernelArg::PointerHwLocal: { - // Calculate current offset in the local ring - uint offset = gpu.cal_.localSize; - uint extra = amd::alignUp(offset, arg->alignment_) - offset; - - offset = amd::alignUp(offset, arg->alignment_); - size_t memSize = *static_cast(param); - - // Allocate new memory from the local ring - gpu.cal_.localSize += static_cast(memSize) + extra; - // Copy current local argument's offset into the CB - *(reinterpret_cast(memory + arg->cbPos_)) = offset; - - CondLog((gpu.cal_.localSize > dev().info().localMemSize_), - "Requested local size is bigger than reported!"); - } break; - case KernelArg::Float: - case KernelArg::Double: - case KernelArg::Char: - case KernelArg::UChar: - case KernelArg::Short: - case KernelArg::UShort: - case KernelArg::Int: - case KernelArg::UInt: - case KernelArg::Long: - case KernelArg::ULong: - if (size != argSize) { - LogWarning("Parameter's sizes are unmatched!"); - } - // Fall through ... - case KernelArg::Struct: - case KernelArg::Union: { - size_t specVec = arg->specialVector(); - if (specVec != 0) { - uint iter = (arg->size_ / KernelArg::VectorSizeLimit); - for (uint i = 0; i < iter; ++i) { - amd::Os::fastMemcpy( - (memory + arg->cbPos_ + i * ConstBuffer::VectorSize), - reinterpret_cast(param) + i * KernelArg::VectorSizeLimit * specVec, - specVec * KernelArg::VectorSizeLimit); - } - } else { - // Copy data into the CB - amd::Os::fastMemcpy((memory + arg->cbPos_), param, size); - } - } break; - default: - LogError("Unhandled argument's type!"); - break; - } - - return result; -} - -bool Kernel::initLocalPrivateRanges(VirtualGPU& gpu) const { - // Initialize HW local - gpu.cal_.localSize = hwLocalSize_; - - // Bind the global buffer if emulated local or private memory - // was allocated by the kernel - if ((flags() & PrintfOutput && (printfId_ == UavIdUndefined)) && (uavRaw_ != UavIdUndefined)) { - if (!bindResource(gpu, dev().globalMem(), 0, GlobalBuffer, uavRaw_)) { - return false; - } - } - - // Bind the global buffer if emulated constant buffers are enabled - if (cbId_ != UavIdUndefined) { - if (!bindResource(gpu, dev().globalMem(), 0, ArgumentCbID, cbId_)) { - return false; - } - } - - // Bind the printf buffer - if (printfId_ != UavIdUndefined) { - if (!bindResource(gpu, dev().globalMem(), 0, ArgumentPrintfID, printfId_)) { - return false; - } - } - // Initialize the iterations count - gpu.cal_.iterations_ = 1; - - return true; -} - -void Kernel::setLocalPrivateRanges(VirtualGPU& gpu) const { - address cbBuf = gpu.cb(0)->sysMemCopy(); - uint* data; - uint gridSize = - gpu.cal()->gridSize.width * gpu.cal()->gridSize.height * gpu.cal()->gridSize.depth; - uint blockSize = - gpu.cal()->gridBlock.width * gpu.cal()->gridBlock.height * gpu.cal()->gridBlock.depth; - - //! \todo validate if the compiler still generates PrivateFixed - if (flags() & PrivateFixed) { - // Update private ring - data = reinterpret_cast(cbBuf + PrivateRingOffset * ConstBuffer::VectorSize); - Memory* gpuMemory = dev().getGpuMemory(dev().dummyPage()); - - if (abi64Bit()) { - reinterpret_cast(data)[0] = gpuMemory->hbOffset(); - data[2] = 0; - data[3] = 0; - } else { - data[0] = static_cast(gpuMemory->hbOffset()); - data[1] = 0; - data[2] = data[3] = 0; - } - gpu.addVmMemory(gpuMemory); - } - - // Copy the math lib constants - amd::Os::fastMemcpy((cbBuf + MathLibOffset * ConstBuffer::VectorSize), MathLibConst, - sizeof(MathLibConst)); - - // Update the offset to the global data - if (prog().glbData() != NULL) { - gpu.addVmMemory(prog().glbData()); - uint64_t glbDataOffset = prog().glbData()->hbOffset(); - if (abi64Bit()) { - *reinterpret_cast(cbBuf + GlobalDataStoreOffset * ConstBuffer::VectorSize) = - glbDataOffset; - } else { - *reinterpret_cast(cbBuf + GlobalDataStoreOffset * ConstBuffer::VectorSize) = - static_cast(glbDataOffset); - } - } - - // Split workload if it was requested - if ((gpu.cal_.iterations_ < 2) && gpu.dmaFlushMgmt().dispatchSplitSize() != 0) { - uint totalSize = gridSize * blockSize; - if (totalSize > gpu.dmaFlushMgmt().dispatchSplitSize()) { - gpu.cal_.iterations_ = - std::max(gpu.cal_.iterations_, (totalSize / gpu.dmaFlushMgmt().dispatchSplitSize())); - } - } - - // Initialize the number of iterations to the grid size - if (flags() & PrintfOutput) { - gpu.cal_.iterations_ = gridSize; - } -} - -void Kernel::setSampler(VirtualGPU& gpu, uint32_t state, uint physUnit) const { - // All CAL sampler's parameters are in floats - float gslAddress = GSL_CLAMP_TO_BORDER; - float gslMinFilter = GSL_MIN_NEAREST; - float gslMagFilter = GSL_MAG_NEAREST; - - state &= ~amd::Sampler::StateNormalizedCoordsMask; - - // Program the sampler address mode - switch (state & amd::Sampler::StateAddressMask) { - case amd::Sampler::StateAddressRepeat: - gslAddress = GSL_REPEAT; - break; - case amd::Sampler::StateAddressClampToEdge: - gslAddress = GSL_CLAMP_TO_EDGE; - break; - case amd::Sampler::StateAddressMirroredRepeat: - gslAddress = GSL_MIRRORED_REPEAT; - break; - case amd::Sampler::StateAddressClamp: - case amd::Sampler::StateAddressNone: - default: - break; - } - state &= ~amd::Sampler::StateAddressMask; - - gpu.setSamplerParameter(physUnit, GSL_TEXTURE_WRAP_S, &gslAddress); - gpu.setSamplerParameter(physUnit, GSL_TEXTURE_WRAP_T, &gslAddress); - gpu.setSamplerParameter(physUnit, GSL_TEXTURE_WRAP_R, &gslAddress); - - // Program texture filter mode - if (state == amd::Sampler::StateFilterLinear) { - gslMinFilter = GSL_MIN_LINEAR; - gslMagFilter = GSL_MAG_LINEAR; - } - - gpu.setSamplerParameter(physUnit, GSL_TEXTURE_MIN_FILTER, &gslMinFilter); - gpu.setSamplerParameter(physUnit, GSL_TEXTURE_MAG_FILTER, &gslMagFilter); -} - -bool Kernel::bindResource(VirtualGPU& gpu, const Memory& memory, uint paramIdx, ResourceType type, - uint physUnit, size_t offset) const { - gslUAVType uavType = GSL_UAV_TYPE_UNKNOWN; - - // Find the original resource name from the IL program - switch (type) { - case GlobalBuffer: - if (gpu.state_.boundGlobal_) { - return true; - } - gpu.state_.boundGlobal_ = true; - physUnit = uavRaw_; - uavType = GSL_UAV_TYPE_TYPELESS; - break; - case ArgumentCbID: - if (gpu.state_.boundCb_) { - return true; - } - gpu.state_.boundCb_ = true; - physUnit = cbId_; - uavType = GSL_UAV_TYPE_TYPELESS; - break; - case ArgumentPrintfID: - if (gpu.state_.boundPrintf_) { - return true; - } - gpu.state_.boundPrintf_ = true; - physUnit = printfId_; - uavType = GSL_UAV_TYPE_TYPELESS; - break; - case ArgumentHeapBuffer: - case ArgumentBuffer: - case ArgumentImageRead: - case ArgumentImageWrite: - case ArgumentConstBuffer: - case ArgumentCounter: - // Early exit if resource is bound already - if (gpu.slots_[paramIdx].state_.bound_) { - return true; - } - - // Associate resource with the slot - gpu.slots_[paramIdx].memory_ = &memory; - - // Mark resource as bound - gpu.slots_[paramIdx].state_.bound_ = true; - - if (type == ArgumentCounter) { - GpuEvent calEvent; - - // Bind memory with atomic counter - gpu.cs()->bindAtomicCounter(argument(paramIdx)->index_, memory.gslResource()); - - // Copy the counter value into GDS - gpu.eventBegin(MainEngine); - gpu.cs()->syncAtomicCounter(argument(paramIdx)->index_, false); - gpu.eventEnd(MainEngine, calEvent); - - // Mark resource as busy - memory.setBusy(gpu, calEvent); - return true; - } else if (type == ArgumentHeapBuffer) { - // We return here, since we just have to bind the global heap - return true; - } else if (type == ArgumentConstBuffer) { - gpu.slots_[paramIdx].state_.constant_ = true; - } - break; - case ArgumentUavID: - case ConstantBuffer: - break; - default: - LogPrintfError("Unspecified argument type ()!", type); - return false; - } - - gslMemObject gslMem = NULL; - // Use global address space on SI+ for UAV setup - if ((type == ArgumentBuffer) || (type == ArgumentCbID) || (type == ArgumentUavID) || - (type == ArgumentPrintfID)) { - gslMem = dev().heap().resource().gslResource(); - } else { - gslMem = memory.gslResource(); - } - - // Associate memory with the physical unit, the actual binding - bool result = true; - switch (type) { - case GlobalBuffer: - case ArgumentBuffer: - case ArgumentImageWrite: - case ArgumentUavID: - case ArgumentCbID: - case ArgumentPrintfID: - if (type == ArgumentImageWrite) { - uavType = GSL_UAV_TYPE_TYPED; - } else if ((type == ArgumentBuffer) || (type == ArgumentUavID)) { - uavType = GSL_UAV_TYPE_TYPELESS; - } - if (gpu.cal_.uavs_[physUnit] != gslMem) { - result = gpu.setUAVBuffer(physUnit, gslMem, uavType); - gpu.setUAVChannelOrder(physUnit, gslMem); - gpu.cal_.uavs_[physUnit] = gslMem; - } - break; - case ConstantBuffer: - case ArgumentConstBuffer: - if ((gpu.cal_.constBuffers_[physUnit] != gslMem) || (offset != 0)) { - result = gpu.setConstantBuffer(physUnit, gslMem, offset, memory.hbSize()); - gpu.cal_.constBuffers_[physUnit] = gslMem; - } - break; - case ArgumentImageRead: - if (gpu.cal_.readImages_[physUnit] != gslMem) { - result = gpu.setInput(physUnit, gslMem); - gpu.cal_.readImages_[physUnit] = gslMem; - } - break; - default: - result = false; - assert(false); - break; - } - if (!result) { - LogPrintfError("setMem failed unit:%d mem:0x%08x!", physUnit, gslMem); - return false; - } - - return true; -} - -void Kernel::unbindResources(VirtualGPU& gpu, GpuEvent calEvent, bool lastRun) const { - // Make sure unbind occurs on the last run, in case the execution had a split - if (lastRun) { - for (uint i = 0; i < arguments_.size(); ++i) { - if (gpu.slots_[i].state_.bound_) { - GpuEvent calEventTmp = calEvent; - - if (KernelArg::Counter == argument(i)->type_) { - // Copy the counter value from GDS - gpu.eventBegin(MainEngine); - gpu.cs()->syncAtomicCounter(argument(i)->index_, true); - gpu.eventEnd(MainEngine, calEventTmp); - } else if (!(gpu.slots_[i].state_.constant_ || argument(i)->memory_.readOnly_)) { - // Signal the abstraction layer that GPU memory is dirty - if (gpu.slots_[i].memory_->owner() != NULL) { - gpu.slots_[i].memory_->owner()->signalWrite(&gpu.dev()); - } - } - // Mark resource as busy - gpu.slots_[i].memory_->setBusy(gpu, calEventTmp); - - gpu.slots_[i].state_.value_ = 0; - } - } - - // Unbind the global buffer - gpu.state_.boundGlobal_ = false; - - // Unbind the constant buffer - gpu.state_.boundCb_ = false; - - // Unbind the pritnf buffer - gpu.state_.boundPrintf_ = false; - } - - // Mark CB busy - for (uint i = 0; i < numCb_; ++i) { - gpu.constBufs_[i]->setBusy(gpu, calEvent); - } - - // Set the event object for the scratch buffer - if (workGroupInfo()->scratchRegs_ > 0) { - dev().scratch(gpu.hwRing())->memObj_->setBusy(gpu, calEvent); - } -} - -void Kernel::copyImageConstants(const amd::Image* amdImage, ImageConstants* imageData) const { - imageData->width_ = static_cast(amdImage->getWidth()); - imageData->height_ = static_cast(amdImage->getHeight()); - imageData->depth_ = static_cast(amdImage->getDepth()); - imageData->dataType_ = static_cast(amdImage->getImageFormat().image_channel_data_type); - - imageData->widthFloat_ = 1.f / static_cast(amdImage->getWidth()); - imageData->heightFloat_ = 1.f / static_cast(amdImage->getHeight()); - imageData->depthFloat_ = 1.f / static_cast(amdImage->getDepth()); - imageData->channelOrder_ = static_cast(amdImage->getImageFormat().image_channel_order); -} - -union MetadataVersion { - struct { - uint64_t revision_ : 16; //!< LLVM metadata revision - uint64_t minorVersion_ : 16; //!< LLVM metadata minor verison - uint64_t majorVersion_ : 16; //!< LLVM metadata major version - }; - uint64_t value_; - MetadataVersion(uint mj, uint mi, uint rev) : value_(0) { - revision_ = rev; - minorVersion_ = mi; - majorVersion_ = mj; - } - MetadataVersion() : value_(0) {} -}; - -//! Version of metadata with buffer attributes -const MetadataVersion MetadataBufferAttributes = MetadataVersion(2, 0, 88); - -//! Version of metadata with type qualifiers -const MetadataVersion MetadataTypeQualifiers = MetadataVersion(3, 1, 103); - -bool NullKernel::parseArguments(const std::string& metaData, uint* uavRefCount) { - // Initialize workgroup info - workGroupInfo_.size_ = nullDev().info().preferredWorkGroupSize_; - MetadataVersion mdVersion; - - // Find first tag - size_t pos = metaData.find(";"); - - // Loop through all provided program arguments - while (pos != std::string::npos) { - KernelArg arg; - - if (!expect(metaData, &pos, ";")) { - break; - } - - arg.type_ = KernelArg::NoType; - - // Loop through all available metadata types - for (uint i = 0; i < ArgStateTotal; ++i) { - uint tmpValue; - // Find the name tag - if (expect(metaData, &pos, ArgState[i].typeName_)) { - switch (ArgState[i].type_) { - case KernelArg::NoType: - // Process next ... - continue; - case KernelArg::Reflection: { - uint argIdx; - // Read the argument's index - if (!getuint(metaData, &pos, &argIdx)) { - LogWarning("Couldn't get the argument index!"); - return false; - } - KernelArg* tmpArg = arguments_[argIdx]; - if (!getstring(metaData, &pos, &tmpArg->typeName_)) { - LogWarning("Couldn't get the argument type!"); - return false; - } - } - continue; - case KernelArg::ConstArg: { - uint argIdx; - // Read the argument's index - if (!getuint(metaData, &pos, &argIdx)) { - LogWarning("Couldn't get the argument index!"); - return false; - } - KernelArg* tmpArg = arguments_[argIdx]; - tmpArg->typeQualifier_ |= CL_KERNEL_ARG_TYPE_CONST; - } - continue; - case KernelArg::Grouping: - for (uint j = 0; j < 3; ++j) { - uint temp; - // Read the compile workgroup size - if (!getuint(metaData, &pos, &temp)) { - LogWarning("Couldn't get the compile workgroup size!"); - return false; - } - workGroupInfo_.compileSize_[j] = temp; - } - // Process next ... - continue; - case KernelArg::WrkgrpSize: { - uint temp; - // Read the workgroup size - if (!getuint(metaData, &pos, &temp)) { - LogWarning("Couldn't get the workgroup size!"); - return false; - } - workGroupInfo_.size_ = temp; - } - // Process next ... - continue; - case KernelArg::Wavefront: - // Process next ... - continue; - case KernelArg::UavId: - // Read index - if (!getuint(metaData, &pos, &arg.index_)) { - return false; - } - break; - case KernelArg::ConstBufId: - // Read index - if (!getuint(metaData, &pos, &cbId_)) { - return false; - } - continue; - case KernelArg::PrintfBufId: - // Read index - if (!getuint(metaData, &pos, &printfId_)) { - return false; - } - continue; - case KernelArg::MetadataVersion: - // Read metadata version - if (!getuint(metaData, &pos, &tmpValue)) { - return false; - } - mdVersion.majorVersion_ = tmpValue; - if (!getuint(metaData, &pos, &tmpValue)) { - return false; - } - mdVersion.minorVersion_ = tmpValue; - if (!getuint(metaData, &pos, &tmpValue)) { - return false; - } - mdVersion.revision_ = tmpValue; - // Process next ... - continue; - case KernelArg::GroupingHint: - for (uint j = 0; j < 3; ++j) { - uint temp; - // Read the compile workgroup size hint - if (!getuint(metaData, &pos, &temp)) { - LogWarning("Couldn't get the compile workgroup size hint!"); - return false; - } - workGroupInfo_.compileSizeHint_[j] = temp; - } - // Process next ... - continue; - case KernelArg::VecTypeHint: { - std::string temp; - // Read the compile vector type hint - if (!getstring(metaData, &pos, &temp)) { - LogWarning("Couldn't get the compile vector type hint!"); - return false; - } - workGroupInfo_.compileVecTypeHint_ = temp; - } - // Process next ... - continue; - case KernelArg::WavesPerSimdHint: { - uint tmp; - if (!getuint(metaData, &pos, &tmp)) { - return false; - } - workGroupInfo_.wavesPerSimdHint_ = tmp; - } - continue; - default: - break; - } - - std::string argName; - // Save the argument type - arg.type_ = ArgState[i].type_; - - // Check if we should expect the name - if (ArgState[i].name_) { - // Read the parameter's name - if (!getword(metaData, &pos, argName)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - arg.name_ = argName; - } - - if (arg.type_ == KernelArg::Sampler) { - if (!getuint(metaData, &pos, &arg.index_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - if (!getuint(metaData, &pos, &arg.location_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - if (!getuint(metaData, &pos, &arg.cbPos_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - } - - // Check if we should expect the resource data type - if (ArgState[i].resType_) { - uint k; - // Search for the data type - for (k = 0; k < DataTypeTotal; k++) { - if (expect(metaData, &pos, DataType[k].tagName_)) { - arg.dataType_ = DataType[k].type_; - if (arg.type_ == KernelArg::Image) { - flags_ |= ImageEnable; - if (expect(metaData, &pos, "RO:")) { - arg.memory_.readOnly_ = 1; - } else if (expect(metaData, &pos, "RW:")) { - arg.memory_.readWrite_ = 1; - flags_ |= ImageWrite; - } else if (expect(metaData, &pos, "WO:")) { - arg.memory_.writeOnly_ = 1; - flags_ |= ImageWrite; - } - } else if (arg.type_ == KernelArg::Value) { - arg.type_ = DataType[k].type_; - } - break; - } - } - if (k == DataTypeTotal) { - LogWarning("We couldn't find the argument's type."); - if ((arg.type_ == KernelArg::Value) || !getword(metaData, &pos, argName)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - } - //! @todo temporary condition - if ((arg.type_ == KernelArg::Opaque) || (arg.type_ == KernelArg::Sampler)) { - assert(false); - continue; - } - } - - // Check if we should expect the data size - if (ArgState[i].size_) { - uint tmpData; - // Read the data size - if (!getuint(metaData, &pos, &tmpData)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - if (arg.type_ == KernelArg::Image) { - arg.type_ = arg.dataType_; - arg.index_ = tmpData; - } else { - arg.size_ = tmpData; - } - } - - if (arg.type_ == KernelArg::Counter) { - // Read a counter index - if (!getuint(metaData, &pos, &arg.index_)) { - LogWarning("Couldn't get a counter index!"); - return false; - } - } - - // Check if we should expect a resource index - if (ArgState[i].cbIdx_) { - // Read resource index - if (!getuint(metaData, &pos, &arg.cbIdx_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - - if (arg.isCbNeeded() && (numCb_ < arg.cbIdx_)) { - numCb_ = arg.cbIdx_; - } - } - // Check if we should expect the CB offset - if (ArgState[i].cbPos_) { - // Read position in the constant buffer - if (!getuint(metaData, &pos, &arg.cbPos_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - } - // Check if we should expect the buffer type - if (ArgState[i].buf_) { - // Read the buffer type - if (!getword(metaData, &pos, argName)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - arg.buf_ = argName; - - for (uint k = 0; k < BufTypeTotal; ++k) { - if (0 == arg.buf_.compare(BufType[k].tagName_)) { - // Update the parameter type - arg.type_ = BufType[k].type_; - // Check if we should expect a buffer index - if (BufType[k].number_) { - // Read a buffer index - if (!getuint(metaData, &pos, &arg.index_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - } - // Check for the required alignment - if (BufType[k].alignment_) { - // Read data alignment - if (!getuint(metaData, &pos, &arg.alignment_)) { - LogWarning("Couldn't get a kernel argument!"); - return false; - } - } - // Check for the buffer's attribute - if ((mdVersion.value_ >= MetadataBufferAttributes.value_) && BufType[k].attribute_) { - if (expect(metaData, &pos, "RO")) { - arg.memory_.readOnly_ = 1; - } else if (expect(metaData, &pos, "RW")) { - arg.memory_.readWrite_ = 1; - } else if (expect(metaData, &pos, "WO")) { - arg.memory_.writeOnly_ = 1; - } - } - // Check for the type qualifier - if ((mdVersion.value_ >= MetadataTypeQualifiers.value_) && BufType[k].attribute_) { - uint tmp; - pos += 1; - if (!getuint(metaData, &pos, &tmp)) { - LogWarning("Couldn't get volatile type!"); - return false; - } - if (tmp == 1) { - arg.typeQualifier_ |= CL_KERNEL_ARG_TYPE_VOLATILE; - } - if (!getuint(metaData, &pos, &tmp)) { - LogWarning("Couldn't get restrict type!"); - return false; - } - if (tmp == 1) { - arg.typeQualifier_ |= CL_KERNEL_ARG_TYPE_RESTRICT; - } - } - } - } - } - // Find multiple UAV references - switch (arg.type_) { - case KernelArg::PointerGlobal: - case KernelArg::PointerConst: - case KernelArg::PointerLocal: - case KernelArg::PointerPrivate: - case KernelArg::UavId: - uavRefCount[arg.index_]++; - break; - default: - break; - } - // Check if this argument will be passed in constant buffer - if (arg.isCbNeeded() || (arg.type_ == KernelArg::UavId)) { - if (arg.type_ == KernelArg::Sampler) { - // Serach for the passed by value sampler - for (uint i = 0; i < argSize(); ++i) { - KernelArg* value = arguments_[i]; - if (0 == value->name_.compare(arg.name_)) { - value->type_ = arg.type_; - value->index_ = arg.index_; - value->location_ = 0; - break; - } - } - } else { - KernelArg* argument = new KernelArg(arg); - if (argument != NULL) { - addArgument(argument); - } else { - LogError("Couldn't allocate memory!"); - return false; - } - } - } - // Check if we have a pre-defined sampler - else if (arg.type_ == KernelArg::Sampler) { - KernelArg* sampler = new KernelArg(arg); - if (sampler != NULL) { - addSampler(sampler); - } else { - LogError("Couldn't allocate memory!"); - return false; - } - } - break; - } - } - - // Next argument - pos = metaData.find(";", pos); - } - - // Find arguments that will require a reallocation - for (uint i = 0; i < arguments_.size(); ++i) { - KernelArg* arg = arguments_[i]; - switch (arg->type_) { - case KernelArg::PointerGlobal: - case KernelArg::PointerConst: - case KernelArg::PointerLocal: - case KernelArg::PointerPrivate: - // Check if can't use a dedicated UAV, - // so realloc memory in the heap - arg->memory_.realloc_ = false; - arg->memory_.uavBuf_ = true; - break; - case KernelArg::PointerHwConst: - arg->memory_.realloc_ = true; - break; - case KernelArg::UavId: - uavRaw_ = arg->index_; - break; - default: - break; - } - // If argument marked with the const qualifier, then overwrite - // Read-Write attributes, since compiler doesn't mark it properly - if (arg->typeQualifier() & CL_KERNEL_ARG_TYPE_CONST) { - arg->memory_.readOnly_ = 1; - arg->memory_.readWrite_ = 0; - arg->memory_.writeOnly_ = 0; - } - } - - if ((uavRaw_ != UavIdUndefined) && !(flags() & PrintfOutput)) { - // Find if default UAV is already assigned to an argument - for (uint i = 0; i < arguments_.size(); ++i) { - KernelArg* arg = arguments_[i]; - switch (arg->type_) { - case KernelArg::PointerGlobal: - case KernelArg::PointerConst: - case KernelArg::PointerLocal: - case KernelArg::PointerPrivate: - if (uavRaw_ == arg->index_) { - uavRaw_ = UavIdUndefined; - } - break; - default: - break; - } - } - } - - // There is always 1 constant buffer, associated with the kernel - numCb_++; - assert((numCb_ <= MaxConstBuffersArguments) && - "Runtime doesn't support more than max CBs for arguments!"); - - // Limit workgroup size if requested - if ((flags() & LimitWorkgroup) && (GPU_MAX_WORKGROUP_SIZE == 0)) { - size_t temp = 1; - workGroupInfo_.size_ = workGroupInfo()->wavefrontSize_; - for (uint j = 0; j < 3; ++j) { - if (workGroupInfo()->compileSize_[j] != 0) { - temp *= workGroupInfo_.compileSize_[j]; - } - } - // Report a compilation error if requested compile size doesn't - // match the required workgroup size - if (workGroupInfo()->size_ < temp) { - char str[8]; - intToStr(workGroupInfo_.size_, str, 8); - buildError_ = CL_OUT_OF_RESOURCES; - buildLog_ += "Error: Requested compile size is bigger than the required workgroup size of "; - buildLog_ += str; - buildLog_ += " elements\n"; - LogError(buildLog().c_str()); - return false; - } - } - - // Read/Write attributes are provided in metadata - if (mdVersion.value_ >= MetadataBufferAttributes.value_) { - rwAttributes_ = true; - } - - return true; -} - -inline static HSAIL_ARG_TYPE GetHSAILArgType(const aclArgData* argInfo) { - switch (argInfo->type) { - case ARG_TYPE_POINTER: - return HSAIL_ARGTYPE_POINTER; - case ARG_TYPE_QUEUE: - return HSAIL_ARGTYPE_QUEUE; - case ARG_TYPE_VALUE: - return HSAIL_ARGTYPE_VALUE; - case ARG_TYPE_IMAGE: - return HSAIL_ARGTYPE_IMAGE; - case ARG_TYPE_SAMPLER: - return HSAIL_ARGTYPE_SAMPLER; - case ARG_TYPE_ERROR: - default: - return HSAIL_ARGTYPE_ERROR; - } -} - -inline static size_t GetHSAILArgAlignment(const aclArgData* argInfo) { - switch (argInfo->type) { - case ARG_TYPE_POINTER: - return argInfo->arg.pointer.align; - default: - return 1; - } -} - -inline static HSAIL_ACCESS_TYPE GetHSAILArgAccessType(const aclArgData* argInfo) { - if (argInfo->type == ARG_TYPE_POINTER) { - switch (argInfo->arg.pointer.type) { - case ACCESS_TYPE_RO: - return HSAIL_ACCESS_TYPE_RO; - case ACCESS_TYPE_WO: - return HSAIL_ACCESS_TYPE_WO; - case ACCESS_TYPE_RW: - default: - return HSAIL_ACCESS_TYPE_RW; - } - } - return HSAIL_ACCESS_TYPE_NONE; -} - -inline static HSAIL_ADDRESS_QUALIFIER GetHSAILAddrQual(const aclArgData* argInfo) { - if (argInfo->type == ARG_TYPE_POINTER) { - switch (argInfo->arg.pointer.memory) { - case PTR_MT_CONSTANT_EMU: - case PTR_MT_CONSTANT: - case PTR_MT_UAV: - case PTR_MT_GLOBAL: - return HSAIL_ADDRESS_GLOBAL; - case PTR_MT_LDS_EMU: - case PTR_MT_LDS: - return HSAIL_ADDRESS_LOCAL; - case PTR_MT_SCRATCH_EMU: - return HSAIL_ADDRESS_GLOBAL; - case PTR_MT_ERROR: - default: - LogError("Unsupported address type"); - return HSAIL_ADDRESS_ERROR; - } - } else if ((argInfo->type == ARG_TYPE_IMAGE) || (argInfo->type == ARG_TYPE_SAMPLER)) { - return HSAIL_ADDRESS_GLOBAL; - } else if (argInfo->type == ARG_TYPE_QUEUE) { - return HSAIL_ADDRESS_GLOBAL; - } - return HSAIL_ADDRESS_ERROR; -} - -/* f16 returns f32 - workaround due to comp lib */ -inline static HSAIL_DATA_TYPE GetHSAILDataType(const aclArgData* argInfo) { - aclArgDataType dataType; - - if (argInfo->type == ARG_TYPE_POINTER) { - dataType = argInfo->arg.pointer.data; - } else if (argInfo->type == ARG_TYPE_VALUE) { - dataType = argInfo->arg.value.data; - } else { - return HSAIL_DATATYPE_ERROR; - } - switch (dataType) { - case DATATYPE_i1: - return HSAIL_DATATYPE_B1; - case DATATYPE_i8: - return HSAIL_DATATYPE_S8; - case DATATYPE_i16: - return HSAIL_DATATYPE_S16; - case DATATYPE_i32: - return HSAIL_DATATYPE_S32; - case DATATYPE_i64: - return HSAIL_DATATYPE_S64; - case DATATYPE_u8: - return HSAIL_DATATYPE_U8; - case DATATYPE_u16: - return HSAIL_DATATYPE_U16; - case DATATYPE_u32: - return HSAIL_DATATYPE_U32; - case DATATYPE_u64: - return HSAIL_DATATYPE_U64; - case DATATYPE_f16: - return HSAIL_DATATYPE_F32; - case DATATYPE_f32: - return HSAIL_DATATYPE_F32; - case DATATYPE_f64: - return HSAIL_DATATYPE_F64; - case DATATYPE_struct: - return HSAIL_DATATYPE_STRUCT; - case DATATYPE_opaque: - return HSAIL_DATATYPE_OPAQUE; - case DATATYPE_ERROR: - default: - return HSAIL_DATATYPE_ERROR; - } -} - -inline static int GetHSAILArgSize(const aclArgData* argInfo) { - switch (argInfo->type) { - case ARG_TYPE_VALUE: - switch (GetHSAILDataType(argInfo)) { - case HSAIL_DATATYPE_B1: - return 1; - case HSAIL_DATATYPE_B8: - case HSAIL_DATATYPE_S8: - case HSAIL_DATATYPE_U8: - return 1; - case HSAIL_DATATYPE_B16: - case HSAIL_DATATYPE_U16: - case HSAIL_DATATYPE_S16: - case HSAIL_DATATYPE_F16: - return 2; - case HSAIL_DATATYPE_B32: - case HSAIL_DATATYPE_U32: - case HSAIL_DATATYPE_S32: - case HSAIL_DATATYPE_F32: - return 4; - case HSAIL_DATATYPE_B64: - case HSAIL_DATATYPE_U64: - case HSAIL_DATATYPE_S64: - case HSAIL_DATATYPE_F64: - return 8; - case HSAIL_DATATYPE_STRUCT: - return argInfo->arg.value.numElements; - default: - return -1; - } - case ARG_TYPE_POINTER: - case ARG_TYPE_IMAGE: - case ARG_TYPE_SAMPLER: - case ARG_TYPE_QUEUE: - return sizeof(void*); - default: - return -1; - } -} - -inline static clk_value_type_t GetOclType(const aclArgData* argInfo) { - static const clk_value_type_t ClkValueMapType[6][6] = { - {T_CHAR, T_CHAR2, T_CHAR3, T_CHAR4, T_CHAR8, T_CHAR16}, - {T_SHORT, T_SHORT2, T_SHORT3, T_SHORT4, T_SHORT8, T_SHORT16}, - {T_INT, T_INT2, T_INT3, T_INT4, T_INT8, T_INT16}, - {T_LONG, T_LONG2, T_LONG3, T_LONG4, T_LONG8, T_LONG16}, - {T_FLOAT, T_FLOAT2, T_FLOAT3, T_FLOAT4, T_FLOAT8, T_FLOAT16}, - {T_DOUBLE, T_DOUBLE2, T_DOUBLE3, T_DOUBLE4, T_DOUBLE8, T_DOUBLE16}, - }; - - uint sizeType; - if (argInfo->type == ARG_TYPE_QUEUE) { - return T_QUEUE; - } - if ((argInfo->type == ARG_TYPE_POINTER) || (argInfo->type == ARG_TYPE_IMAGE)) { - return T_POINTER; - } else if (argInfo->type == ARG_TYPE_VALUE) { - switch (argInfo->arg.value.data) { - case DATATYPE_i8: - case DATATYPE_u8: - sizeType = 0; - break; - case DATATYPE_i16: - case DATATYPE_u16: - sizeType = 1; - break; - case DATATYPE_i32: - case DATATYPE_u32: - sizeType = 2; - break; - case DATATYPE_i64: - case DATATYPE_u64: - sizeType = 3; - break; - case DATATYPE_f16: - case DATATYPE_f32: - sizeType = 4; - break; - case DATATYPE_f64: - sizeType = 5; - break; - default: - return T_VOID; - } - switch (argInfo->arg.value.numElements) { - case 1: - return ClkValueMapType[sizeType][0]; - case 2: - return ClkValueMapType[sizeType][1]; - case 3: - return ClkValueMapType[sizeType][2]; - case 4: - return ClkValueMapType[sizeType][3]; - case 8: - return ClkValueMapType[sizeType][4]; - case 16: - return ClkValueMapType[sizeType][5]; - default: - return T_VOID; - } - } else if (argInfo->type == ARG_TYPE_SAMPLER) { - return T_SAMPLER; - } else { - return T_VOID; - } -} - -inline static cl_kernel_arg_address_qualifier GetOclAddrQual(const aclArgData* argInfo) { - if (argInfo->type == ARG_TYPE_POINTER) { - switch (argInfo->arg.pointer.memory) { - case PTR_MT_UAV: - case PTR_MT_GLOBAL: - return CL_KERNEL_ARG_ADDRESS_GLOBAL; - case PTR_MT_CONSTANT: - case PTR_MT_UAV_CONSTANT: - case PTR_MT_CONSTANT_EMU: - return CL_KERNEL_ARG_ADDRESS_CONSTANT; - case PTR_MT_LDS_EMU: - case PTR_MT_LDS: - return CL_KERNEL_ARG_ADDRESS_LOCAL; - default: - return CL_KERNEL_ARG_ADDRESS_PRIVATE; - } - } else if (argInfo->type == ARG_TYPE_IMAGE) { - return CL_KERNEL_ARG_ADDRESS_GLOBAL; - } - // default for all other cases - return CL_KERNEL_ARG_ADDRESS_PRIVATE; -} - -inline static cl_kernel_arg_access_qualifier GetOclAccessQual(const aclArgData* argInfo) { - if (argInfo->type == ARG_TYPE_IMAGE) { - switch (argInfo->arg.image.type) { - case ACCESS_TYPE_RO: - return CL_KERNEL_ARG_ACCESS_READ_ONLY; - case ACCESS_TYPE_WO: - return CL_KERNEL_ARG_ACCESS_WRITE_ONLY; - case ACCESS_TYPE_RW: - return CL_KERNEL_ARG_ACCESS_READ_WRITE; - default: - return CL_KERNEL_ARG_ACCESS_NONE; - } - } - return CL_KERNEL_ARG_ACCESS_NONE; -} - -inline static cl_kernel_arg_type_qualifier GetOclTypeQual(const aclArgData* argInfo) { - cl_kernel_arg_type_qualifier rv = CL_KERNEL_ARG_TYPE_NONE; - if (argInfo->type == ARG_TYPE_POINTER) { - if (argInfo->arg.pointer.isVolatile) { - rv |= CL_KERNEL_ARG_TYPE_VOLATILE; - } - if (argInfo->arg.pointer.isRestrict) { - rv |= CL_KERNEL_ARG_TYPE_RESTRICT; - } - if (argInfo->arg.pointer.isPipe) { - rv |= CL_KERNEL_ARG_TYPE_PIPE; - } - if (argInfo->isConst) { - rv |= CL_KERNEL_ARG_TYPE_CONST; - } - switch (argInfo->arg.pointer.memory) { - case PTR_MT_CONSTANT: - case PTR_MT_UAV_CONSTANT: - case PTR_MT_CONSTANT_EMU: - rv |= CL_KERNEL_ARG_TYPE_CONST; - break; - default: - break; - } - } - return rv; -} - -static int GetOclSize(const aclArgData* argInfo) { - switch (argInfo->type) { - case ARG_TYPE_POINTER: - return sizeof(void*); - case ARG_TYPE_VALUE: - //! \note OCL 6.1.5. For 3-component vector data types, - //! the size of the data type is 4 * sizeof(component). - switch (argInfo->arg.value.data) { - case DATATYPE_struct: - return 1 * argInfo->arg.value.numElements; - case DATATYPE_i8: - case DATATYPE_u8: - return 1 * amd::nextPowerOfTwo(argInfo->arg.value.numElements); - case DATATYPE_u16: - case DATATYPE_i16: - case DATATYPE_f16: - return 2 * amd::nextPowerOfTwo(argInfo->arg.value.numElements); - case DATATYPE_u32: - case DATATYPE_i32: - case DATATYPE_f32: - return 4 * amd::nextPowerOfTwo(argInfo->arg.value.numElements); - case DATATYPE_i64: - case DATATYPE_u64: - case DATATYPE_f64: - return 8 * amd::nextPowerOfTwo(argInfo->arg.value.numElements); - case DATATYPE_ERROR: - default: - return -1; - } - case ARG_TYPE_IMAGE: - return sizeof(cl_mem); - case ARG_TYPE_SAMPLER: - return sizeof(cl_sampler); - case ARG_TYPE_QUEUE: - return sizeof(cl_command_queue); - default: - return -1; - } -} - -void HSAILKernel::initArgList(const aclArgData* aclArg) { - // Initialize the hsail argument list too - initHsailArgs(aclArg); - - // Iterate through the arguments and insert into parameterList - device::Kernel::parameters_t params; - amd::KernelParameterDescriptor desc; - size_t offset = 0; - - // Reserved arguments for HSAIL launch - aclArg += MaxExtraArgumentsNum; - for (uint i = 0; aclArg->struct_size != 0; i++, aclArg++) { - desc.name_ = arguments_[i]->name_.c_str(); - desc.type_ = GetOclType(aclArg); - desc.addressQualifier_ = GetOclAddrQual(aclArg); - desc.accessQualifier_ = GetOclAccessQual(aclArg); - desc.typeQualifier_ = GetOclTypeQual(aclArg); - desc.typeName_ = arguments_[i]->typeName_.c_str(); - - // Make a check if it is local or global - if (desc.addressQualifier_ == CL_KERNEL_ARG_ADDRESS_LOCAL) { - desc.size_ = sizeof(cl_mem); - } else { - desc.size_ = GetOclSize(aclArg); - } - - // Make offset alignment to match CPU metadata, since - // in multidevice config abstraction layer has a single signature - // and CPU sends the paramaters as they are allocated in memory - size_t size = desc.size_; - - offset = amd::alignUp(offset, std::min(size, size_t(16))); - desc.offset_ = offset; - offset += amd::alignUp(size, sizeof(uint32_t)); - params.push_back(desc); - - if (arguments_[i]->type_ == HSAIL_ARGTYPE_IMAGE) { - flags_.imageEna_ = true; - if (desc.accessQualifier_ != CL_KERNEL_ARG_ACCESS_READ_ONLY) { - flags_.imageWriteEna_ = true; - } - } - } - - createSignature(params, params.size(), amd::KernelSignature::ABIVersion_0); -} - -void HSAILKernel::initHsailArgs(const aclArgData* aclArg) { - int offset = 0; - - // Reserved arguments for HSAIL launch - aclArg += MaxExtraArgumentsNum; - - // Iterate through the each kernel argument - for (; aclArg->struct_size != 0; aclArg++) { - Argument* arg = new Argument; - // Initialize HSAIL kernel argument - arg->name_ = aclArg->argStr; - arg->typeName_ = aclArg->typeStr; - arg->size_ = GetHSAILArgSize(aclArg); - arg->offset_ = offset; - arg->type_ = GetHSAILArgType(aclArg); - arg->addrQual_ = GetHSAILAddrQual(aclArg); - arg->dataType_ = GetHSAILDataType(aclArg); - // If vector of args we add additional arguments to flatten it out - arg->numElem_ = - ((aclArg->type == ARG_TYPE_VALUE) && (aclArg->arg.value.data != DATATYPE_struct)) - ? aclArg->arg.value.numElements - : 1; - arg->alignment_ = GetHSAILArgAlignment(aclArg); - arg->access_ = GetHSAILArgAccessType(aclArg); - offset += GetHSAILArgSize(aclArg); - arguments_.push_back(arg); - } -} - -HSAILKernel::HSAILKernel(std::string name, HSAILProgram* prog, std::string compileOptions, - uint extraArgsNum) - : device::Kernel(prog->device(), name, *prog), - compileOptions_(compileOptions), - index_(0), - code_(NULL), - codeSize_(0), - hwMetaData_(NULL), - extraArgumentsNum_(extraArgsNum) { - flags_.hsa_ = true; -} - -HSAILKernel::~HSAILKernel() { - while (!arguments_.empty()) { - Argument* arg = arguments_.back(); - delete arg; - arguments_.pop_back(); - } - - delete[] hwMetaData_; - - delete code_; -} - -bool HSAILKernel::init(amd::hsa::loader::Symbol* sym, bool finalize) { - if (extraArgumentsNum_ > MaxExtraArgumentsNum) { - LogError("Failed to initialize kernel: extra arguments number is bigger than is supported"); - return false; - } - acl_error error = ACL_SUCCESS; - std::string openClKernelName = openclMangledName(name()); - flags_.internalKernel_ = - (compileOptions_.find("-cl-internal-kernel") != std::string::npos) ? true : false; - // compile kernel down to ISA - if (finalize) { - std::string options(compileOptions_.c_str()); - options.append(" -just-kernel="); - options.append(openClKernelName.c_str()); - // Append an option so that we can selectively enable a SCOption on CZ - // whenever IOMMUv2 is enabled. - if (dev().settings().svmFineGrainSystem_) { - options.append(" -sc-xnack-iommu"); - } - error = amd::Hsail::Compile(dev().hsaCompiler(), prog().binaryElf(), options.c_str(), ACL_TYPE_CG, - ACL_TYPE_ISA, NULL); - buildLog_ += amd::Hsail::GetCompilerLog(dev().hsaCompiler()); - if (error != ACL_SUCCESS) { - LogError("Failed to finalize kernel"); - return false; - } - } - - if (!aqlCreateHWInfo(sym)) { - return false; - } - - // Pull out metadata from the ELF - size_t sizeOfArgList; - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_ARGUMENT_ARRAY, - openClKernelName.c_str(), NULL, &sizeOfArgList); - if (error != ACL_SUCCESS) { - return false; - } - - char* aclArgList = new char[sizeOfArgList]; - if (NULL == aclArgList) { - return false; - } - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_ARGUMENT_ARRAY, - openClKernelName.c_str(), aclArgList, &sizeOfArgList); - if (error != ACL_SUCCESS) { - return false; - } - - size_t sizeOfWorkGroupSize; - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_WORK_GROUP_SIZE, - openClKernelName.c_str(), NULL, &sizeOfWorkGroupSize); - if (error != ACL_SUCCESS) { - return false; - } - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_WORK_GROUP_SIZE, - openClKernelName.c_str(), workGroupInfo_.compileSize_, &sizeOfWorkGroupSize); - if (error != ACL_SUCCESS) { - return false; - } - - // Copy wavefront size - workGroupInfo_.wavefrontSize_ = prog().isNull() ? 64 : dev().getAttribs().wavefrontSize; - - // Find total workgroup size - if (workGroupInfo_.compileSize_[0] != 0) { - workGroupInfo_.size_ = workGroupInfo_.compileSize_[0] * workGroupInfo_.compileSize_[1] * - workGroupInfo_.compileSize_[2]; - } else { - workGroupInfo_.size_ = dev().info().preferredWorkGroupSize_; - } - - // Pull out printf metadata from the ELF - size_t sizeOfPrintfList; - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_GPU_PRINTF_ARRAY, - openClKernelName.c_str(), NULL, &sizeOfPrintfList); - if (error != ACL_SUCCESS) { - return false; - } - - // Make sure kernel has any printf info - if (0 != sizeOfPrintfList) { - char* aclPrintfList = new char[sizeOfPrintfList]; - if (NULL == aclPrintfList) { - return false; - } - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_GPU_PRINTF_ARRAY, - openClKernelName.c_str(), aclPrintfList, &sizeOfPrintfList); - if (error != ACL_SUCCESS) { - return false; - } - - // Set the PrintfList - InitPrintf(reinterpret_cast(aclPrintfList)); - delete[] aclPrintfList; - } - - aclMetadata md; - md.enqueue_kernel = false; - size_t sizeOfDeviceEnqueue = sizeof(md.enqueue_kernel); - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_DEVICE_ENQUEUE, - openClKernelName.c_str(), &md.enqueue_kernel, &sizeOfDeviceEnqueue); - if (error != ACL_SUCCESS) { - return false; - } - flags_.dynamicParallelism_ = md.enqueue_kernel; - - md.kernel_index = -1; - size_t sizeOfIndex = sizeof(md.kernel_index); - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_KERNEL_INDEX, - openClKernelName.c_str(), &md.kernel_index, &sizeOfIndex); - if (error != ACL_SUCCESS) { - return false; - } - index_ = md.kernel_index; - - size_t sizeOfWavesPerSimdHint = sizeof(workGroupInfo_.wavesPerSimdHint_); - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_WAVES_PER_SIMD_HINT, - openClKernelName.c_str(), &workGroupInfo_.wavesPerSimdHint_, - &sizeOfWavesPerSimdHint); - if (error != ACL_SUCCESS) { - return false; - } - - waveLimiter_.enable(dev().settings().ciPlus_); - - size_t sizeOfWorkGroupSizeHint = sizeof(workGroupInfo_.compileSizeHint_); - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_WORK_GROUP_SIZE_HINT, - openClKernelName.c_str(), workGroupInfo_.compileSizeHint_, - &sizeOfWorkGroupSizeHint); - if (error != ACL_SUCCESS) { - return false; - } - - size_t sizeOfVecTypeHint; - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_VEC_TYPE_HINT, - openClKernelName.c_str(), NULL, &sizeOfVecTypeHint); - if (error != ACL_SUCCESS) { - return false; - } - - if (0 != sizeOfVecTypeHint) { - char* VecTypeHint = new char[sizeOfVecTypeHint + 1]; - if (NULL == VecTypeHint) { - return false; - } - error = amd::Hsail::QueryInfo(dev().hsaCompiler(), prog().binaryElf(), RT_VEC_TYPE_HINT, - openClKernelName.c_str(), VecTypeHint, &sizeOfVecTypeHint); - if (error != ACL_SUCCESS) { - return false; - } - VecTypeHint[sizeOfVecTypeHint] = '\0'; - workGroupInfo_.compileVecTypeHint_ = std::string(VecTypeHint); - delete[] VecTypeHint; - } - - // Set the argList - initArgList(reinterpret_cast(aclArgList)); - delete[] aclArgList; - - return true; -} - -const Device& HSAILKernel::dev() const { return reinterpret_cast(dev_); } - -const HSAILProgram& HSAILKernel::prog() const { - return reinterpret_cast(prog_); -} - -inline static void WriteAqlArg( - unsigned char** dst, //!< The write pointer to the buffer - const void* src, //!< The source pointer - uint size, //!< The size in bytes to copy - uint alignment = 0 //!< The alignment to follow while writing to the buffer - ) { - if (alignment == 0) { - *dst = amd::alignUp(*dst, size); - } else { - *dst = amd::alignUp(*dst, alignment); - } - memcpy(*dst, src, size); - *dst += size; -} - -const uint16_t kDispatchPacketHeader = (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | - (1 << HSA_PACKET_HEADER_BARRIER) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | - (HSA_FENCE_SCOPE_AGENT << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); - -hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( - VirtualGPU& gpu, const amd::Kernel& kernel, const amd::NDRangeContainer& sizes, - const_address parameters, bool nativeMem, uint64_t vmDefQueue, uint64_t* vmParentWrap, - std::vector& memList) const { - static const bool WaitOnBusyEngine = true; - uint64_t ldsAddress = ldsSize(); - address aqlArgBuf = gpu.cb(0)->sysMemCopy(); - address aqlStruct = gpu.cb(1)->sysMemCopy(); - bool srdResource = false; - - if (extraArgumentsNum_ > 0) { - assert(MaxExtraArgumentsNum >= 6 && - "MaxExtraArgumentsNum has changed, the below algorithm should be changed accordingly"); - size_t extraArgs[MaxExtraArgumentsNum] = {0, 0, 0, 0, 0, 0}; - // The HLC generates up to 3 additional arguments for the global offsets - for (uint i = 0; i < sizes.dimensions(); ++i) { - extraArgs[i] = sizes.offset()[i]; - } - // Check if the kernel may have printf output - if ((printfInfo().size() > 0) && - // and printf buffer was allocated - (gpu.printfDbgHSA().dbgBuffer() != NULL)) { - // and set the fourth argument as the printf_buffer pointer - extraArgs[3] = static_cast(gpu.printfDbgHSA().dbgBuffer()->vmAddress()); - memList.push_back(gpu.printfDbgHSA().dbgBuffer()); - } - if (dynamicParallelism()) { - // Provide the host parent AQL wrap object to the kernel - AmdAqlWrap* wrap = reinterpret_cast(aqlStruct); - memset(wrap, 0, sizeof(AmdAqlWrap)); - wrap->state = AQL_WRAP_BUSY; - ConstBuffer* cb = gpu.constBufs_[1]; - cb->uploadDataToHw(sizeof(AmdAqlWrap)); - *vmParentWrap = cb->vmAddress() + cb->wrtOffset(); - // and set 5th & 6th arguments - extraArgs[4] = vmDefQueue; - extraArgs[5] = *vmParentWrap; - memList.push_back(cb); - } - WriteAqlArg(&aqlArgBuf, extraArgs, sizeof(size_t) * extraArgumentsNum_, sizeof(size_t)); - } - - const amd::KernelSignature& signature = kernel.signature(); - const amd::KernelParameters& kernelParams = kernel.parameters(); - - amd::Memory* const* memories = - reinterpret_cast(parameters + kernelParams.memoryObjOffset()); - - // Find all parameters for the current kernel - for (uint i = 0; i != signature.numParameters(); ++i) { - const HSAILKernel::Argument* arg = argument(i); - const amd::KernelParameterDescriptor& desc = signature.at(i); - const_address paramaddr = parameters + desc.offset_; - - switch (arg->type_) { - case HSAIL_ARGTYPE_POINTER: - // If it is a global pointer - if (arg->addrQual_ == HSAIL_ADDRESS_GLOBAL) { - Memory* gpuMem = NULL; - amd::Memory* mem = NULL; - - uint32_t index = signature.at(i).info_.arrayIndex_; - if (nativeMem) { - gpuMem = reinterpret_cast(memories)[index]; - if (nullptr != gpuMem) { - mem = gpuMem->owner(); - } - } else { - mem = memories[index]; - if (mem != nullptr) { - gpuMem = dev().getGpuMemory(mem); - } - } - - WriteAqlArg(&aqlArgBuf, paramaddr, sizeof(paramaddr), sizeof(paramaddr)); - if (gpuMem == nullptr) { - break; - } - - // Wait for resource if it was used on an inactive engine - //! \note syncCache may call DRM transfer - gpuMem->wait(gpu, WaitOnBusyEngine); - - //! @todo Compiler has to return read/write attributes - if ((NULL != mem) && ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0)) { - mem->signalWrite(&dev()); - } - memList.push_back(gpuMem); - - // save the memory object pointer to allow global memory access - if (NULL != dev().hwDebugMgr()) { - dev().hwDebugMgr()->assignKernelParamMem(i, gpuMem->owner()); - } - } - // If it is a local pointer - else { - assert((arg->addrQual_ == HSAIL_ADDRESS_LOCAL) && "Unsupported address type"); - ldsAddress = amd::alignUp(ldsAddress, arg->alignment_); - WriteAqlArg(&aqlArgBuf, &ldsAddress, desc.size_); - if (desc.size_ == 8) { - ldsAddress += *reinterpret_cast(paramaddr); - } else { - ldsAddress += *reinterpret_cast(paramaddr); - } - } - break; - case HSAIL_ARGTYPE_VALUE: - // Special case for structrues - if (arg->dataType_ == HSAIL_DATATYPE_STRUCT) { - // Copy the current structre into CB1 - memcpy(aqlStruct, paramaddr, arg->size_); - ConstBuffer* cb = gpu.constBufs_[1]; - cb->uploadDataToHw(arg->size_); - // Then use a pointer in aqlArgBuffer to CB1 - uint64_t gpuPtr = cb->vmAddress() + cb->wrtOffset(); - WriteAqlArg(&aqlArgBuf, &gpuPtr, sizeof(void*)); - memList.push_back(cb); - } else { - WriteAqlArg(&aqlArgBuf, paramaddr, arg->numElem_ * arg->size_, arg->size_); - } - break; - case HSAIL_ARGTYPE_IMAGE: { - Image* image = nullptr; - amd::Memory* mem = nullptr; - uint32_t index = signature.at(i).info_.arrayIndex_; - if (nativeMem) { - image = reinterpret_cast(memories)[index]; - if (nullptr != image) { - mem = image->owner(); - } - } else { - mem = memories[index]; - if (mem == NULL) { - LogError("The kernel image argument isn't an image object!"); - return nullptr; - } - image = static_cast(dev().getGpuMemory(mem)); - } - - // Wait for resource if it was used on an inactive engine - //! \note syncCache may call DRM transfer - image->wait(gpu, WaitOnBusyEngine); - - //! \note Special case for the image views. - //! Copy SRD to CB1, so blit manager will be able to release - //! this view without a wait for SRD resource. - if (image->memoryType() == Resource::ImageView) { - // Copy the current structre into CB1 - memcpy(aqlStruct, image->hwState(), HsaImageObjectSize); - ConstBuffer* cb = gpu.constBufs_[1]; - cb->uploadDataToHw(HsaImageObjectSize); - // Then use a pointer in aqlArgBuffer to CB1 - uint64_t srd = cb->vmAddress() + cb->wrtOffset(); - WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); - memList.push_back(cb); - } else { - uint64_t srd = image->hwSrd(); - WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); - srdResource = true; - } - - //! @todo Compiler has to return read/write attributes - if ((NULL != mem) && ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0)) { - mem->signalWrite(&dev()); - } - - memList.push_back(image); - break; - } - case HSAIL_ARGTYPE_SAMPLER: { - uint32_t index = signature.at(i).info_.arrayIndex_; - const amd::Sampler* sampler = reinterpret_cast(parameters + - kernelParams.samplerObjOffset())[index]; - const Sampler* gpuSampler = static_cast(sampler->getDeviceSampler(dev())); - uint64_t srd = gpuSampler->hwSrd(); - WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); - srdResource = true; - break; - } - case HSAIL_ARGTYPE_QUEUE: { - uint32_t index = signature.at(i).info_.arrayIndex_; - const amd::DeviceQueue* queue = reinterpret_cast( - parameters + kernelParams.queueObjOffset())[index]; - VirtualGPU* gpuQueue = static_cast(queue->vDev()); - uint64_t vmQueue; - if (dev().settings().useDeviceQueue_) { - vmQueue = gpuQueue->vQueue()->vmAddress(); - } else { - if (!gpu.createVirtualQueue(queue->size())) { - LogError("Virtual queue creation failed!"); - return nullptr; - } - vmQueue = gpu.vQueue()->vmAddress(); - } - WriteAqlArg(&aqlArgBuf, &vmQueue, sizeof(void*)); - break; - } - default: - LogError(" Unsupported address type "); - return NULL; - } - } - - if (ldsAddress > dev().info().localMemSize_) { - LogError("No local memory available\n"); - return NULL; - } - - // HSAIL kernarg segment size is rounded up to multiple of 16. - aqlArgBuf = amd::alignUp(aqlArgBuf, 16); - assert((aqlArgBuf == (gpu.cb(0)->sysMemCopy() + argsBufferSize())) && - "Size and the number of arguments don't match!"); - hsa_kernel_dispatch_packet_t* hsaDisp = - reinterpret_cast(aqlArgBuf); - - amd::NDRange local(sizes.local()); - const amd::NDRange& global = sizes.global(); - - // Check if runtime has to find local workgroup size - FindLocalWorkSize(sizes.dimensions(), sizes.global(), local); - - hsaDisp->header = kDispatchPacketHeader; - hsaDisp->setup = sizes.dimensions(); - - hsaDisp->workgroup_size_x = local[0]; - hsaDisp->workgroup_size_y = (sizes.dimensions() > 1) ? local[1] : 1; - hsaDisp->workgroup_size_z = (sizes.dimensions() > 2) ? local[2] : 1; - - hsaDisp->grid_size_x = global[0]; - hsaDisp->grid_size_y = (sizes.dimensions() > 1) ? global[1] : 1; - hsaDisp->grid_size_z = (sizes.dimensions() > 2) ? global[2] : 1; - hsaDisp->reserved2 = 0; - - // Initialize kernel ISA and execution buffer requirements - hsaDisp->private_segment_size = spillSegSize(); - hsaDisp->group_segment_size = ldsAddress; - hsaDisp->kernel_object = gpuAqlCode()->vmAddress(); - - ConstBuffer* cb = gpu.constBufs_[0]; - cb->uploadDataToHw(argsBufferSize() + sizeof(hsa_kernel_dispatch_packet_t)); - uint64_t argList = cb->vmAddress() + cb->wrtOffset(); - - hsaDisp->kernarg_address = reinterpret_cast(argList); - hsaDisp->reserved2 = 0; - hsaDisp->completion_signal.handle = 0; - - memList.push_back(cb); - memList.push_back(gpuAqlCode()); - for (gpu::Memory* mem : prog().globalStores()) { - memList.push_back(mem); - } - if (AMD_HSA_BITS_GET(cpuAqlCode_->kernel_code_properties, - AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_QUEUE_PTR)) { - memList.push_back(gpu.hsaQueueMem()); - } - - if (srdResource || prog().isStaticSampler()) { - dev().srds().fillResourceList(memList); - } - - return hsaDisp; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpukernel.hpp b/projects/clr/rocclr/device/gpu/gpukernel.hpp deleted file mode 100644 index f637f9f87f..0000000000 --- a/projects/clr/rocclr/device/gpu/gpukernel.hpp +++ /dev/null @@ -1,877 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUKERNEL_HPP_ -#define GPUKERNEL_HPP_ - -#include "device/device.hpp" -#include "utils/macros.hpp" -#include "platform/command.hpp" -#include "platform/program.hpp" -#include "platform/kernel.hpp" -#include "platform/sampler.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "amd_hsa_kernel_code.h" -#include "device/gpu/gpuprintf.hpp" -#include "device/devwavelimiter.hpp" -#include "hsa.h" - -namespace amd { -namespace hsa { -namespace loader { -class Symbol; -} // loader -} // hsa -} // amd - -//! \namespace gpu GPU Device Implementation -namespace gpu { - -class VirtualGPU; -class Device; -class NullDevice; -class HSAILProgram; - -struct HWSHADER_Helper { - template static T Get(S base, T offset) { - return reinterpret_cast(reinterpret_cast(base) + reinterpret_cast(offset)); - } -}; - -#define HWSHADER_Get(shader, field) HWSHADER_Helper::Get((shader), (shader)->field) - -template -static void CalcPtr(D& dst, const S src, size_t structSize, size_t size) { - dst = reinterpret_cast(reinterpret_cast(src) + structSize * size); -} - -/*! \addtogroup GPU GPU Device Implementation - * @{ - */ - -/*! \brief Helper function for the std::string processing. - * Finds the name in the std::string - * - * \return True if we found the entry of the symbols - */ -bool expect(const std::string& str, //!< The original std::string - size_t* pos, //!< Position to start - const std::string& sym //!< The sympols to expect - ); - -/*! \brief Helper function for the std::string processing. - * Gets a word from the std::string - * - * \return True if we successfully received a word - */ -bool getword(const std::string& str, //!< The original std::string - size_t* pos, //!< Position to start - std::string& sym //!< Returned word - ); - -/*! \brief Helper function for the std::string processing. - * Loads numbers from the metadata - * - * \return True if we loaded a number - */ -bool getuint(const std::string& str, //!< The original std::string - size_t* pos, //!< Position to start - uint* val //!< Returned number - ); - -/*! \brief Helper function for the std::string processing. - * Loads numbers from the metadata in HEX format - * - * \return True if we loaded a number - */ -bool getuintHex(const std::string& str, //!< The original std::string - size_t* pos, //!< Position to start - uint* val //!< Returned number - ); - -/*! \brief Helper function for the std::string processing. - * Loads numbers from the metadata in HEX format - * - * \return True if we loaded a number - */ -bool getuint64Hex(const std::string& str, //!< The original std::string - size_t* pos, //!< Position to start - uint64_t* val //!< Returned number - ); - -/*! \brief Helper function for the std::string processing. - * Converts unsigned integer to string - * - * \return None - */ -void intToStr(size_t value, //!< Value for conversion - char* str, //!< Pointer to the converted string - size_t size //!< String size - ); - -//! Image constant data from ABI specification -struct ImageConstants : public amd::EmbeddedObject { - uint32_t width_; //!< Image surface width - uint32_t height_; //!< Image surface height - uint32_t depth_; //!< Image surface depth (1 for 2D images) - uint32_t dataType_; //!< Image surface data type - float widthFloat_; //!< Image surface width - float heightFloat_; //!< Image surface height - float depthFloat_; //!< Image surface depth (1 for 2D images) - uint32_t channelOrder_; //!< Image surface texels channel order -}; - -//! Kernel arguments -struct KernelArg : public amd::HeapObject { - public: - //! \enum Kernel argument type - enum ArgumentType { - NoType = 0, - PointerGlobal, - Value, - Image, - PointerLocal, - PointerHwLocal, - PointerPrivate, - PointerHwPrivate, - PointerConst, - PointerHwConst, - Float, - Double, - Half, - Char, - UChar, - Short, - UShort, - Int, - UInt, - Long, - ULong, - Struct, - Union, - Opaque, - Event, - Image1D, //!< first image - Image2D, - Image1DB, - Image1DA, - Image2DA, - Image3D, //!< last image - Counter, - Sampler, - PrivateSize, - LocalSize, - HwPrivateSize, - HwLocalSize, - Grouping, - WrkgrpSize, - Wavefront, - PrivateFixed, - ErrorMessage, - WarningMessage, - PrintfFormatStr, - MetadataVersion, - UavId, - ABI64Bit, - GWS, - SWGWS, - Reflection, - ConstArg, - ConstBufId, - PrintfBufId, - GroupingHint, - VecTypeHint, - WavesPerSimdHint, - TotalTypes - }; - - // The compiler metadata fields - std::string name_; //!< parameters name - ArgumentType type_; //!< type of argument - union { - uint size_; //!< number of arguments (for values and pointers only) - uint location_; //!< sampler's location (for samplers only) - }; - uint cbIdx_; //!< constant buffer index - uint cbPos_; //!< dword address in CB for the argument - std::string buf_; //!< buffer tag - uint index_; //!< buffer/image/sampler index - uint alignment_; //!< the required argument's alignment - ArgumentType dataType_; //!< data type of the argument - union { - struct { - uint uavBuf_ : 1; //!< UAV memory, no global heap - uint realloc_ : 1; //!< argument has to be reallocatedin the global heap - uint readOnly_ : 1; //!< Read only memory object - uint writeOnly_ : 1; //!< Write only memory object - uint readWrite_ : 1; //!< Read/Write memory object - }; - uint value_; - } memory_; - - std::string typeName_; //!< argument's type name - uint typeQualifier_; //!< argument's type qualifier - - //! Default constructor for the kernel argument - KernelArg(); - - //! Copy constructor for the kernel argument - KernelArg(const KernelArg& data); - - //! Overloads operator= - KernelArg& operator=(const KernelArg& data); - - //! Destructor of the kernel argument - ~KernelArg() { name_.clear(); } - - /*! \brief Checks if this arguments requires a place in constant buffer - * - * \return True if we need CB - */ - bool isCbNeeded() const; - - /*! \brief Retrieves the argument's size - * - * \return Size of the current argument - */ - size_t size(bool gpuLayer //!< True if we want the argument's size for the GPU layer - ) const; - - /*! \brief Retrieves the argument's type for the abstraction layer - * - * \return The argument's type in the abstraction layer format - */ - clk_value_type_t type() const; - - /*! \brief Retrieves the argument's address qualifier for the abstraction layer - * - * \return The argument's address qualifier in the abstraction layer format - */ - cl_kernel_arg_address_qualifier addressQualifier() const; - - /*! \brief Retrieves the argument's access qualifier for the abstraction layer - * - * \return The argument's access qualifier in the abstraction layer format - */ - cl_kernel_arg_access_qualifier accessQualifier() const; - - /*! \brief Retrieves the argument's type name for the abstraction layer - * - * \return The argument's type name - */ - const char* typeName() const { return typeName_.c_str(); } - - /*! \brief Retrieves the argument's type qualifier for the abstraction layer - * - * \return The argument's type qualifier - */ - cl_kernel_arg_type_qualifier typeQualifier() const { - switch (type_) { - case PointerConst: - case PointerHwConst: - return static_cast(typeQualifier_ | CL_KERNEL_ARG_TYPE_CONST); - default: - return static_cast(typeQualifier_); - } - } - - //! Special case for vectors with component size <= 16bit - static constexpr uint VectorSizeLimit = 4; - size_t specialVector() const; -}; - -struct DataTypeConst { - const char* tagName_; //!< data type's name - KernelArg::ArgumentType type_; //!< data type -}; - -//! Metadata description for parsing -struct MetaDataConst { - const char* typeName_; //!< parameters name - KernelArg::ArgumentType type_; //!< type of argument - struct { - uint size_ : 1; //!< number of arguments - uint name_ : 1; //!< argument's name - uint resType_ : 1; //!< argument's type - uint cbIdx_ : 1; //!< resource index CB, sampler or image - uint cbPos_ : 1; //!< dword address in CB for the argument - uint buf_ : 1; //!< buffer tag - uint reserved : 26; //!< reserved - }; -}; - -const uint DescTotal = 15; -const uint BasicTypeTotal = 15; -const uint ArgStateTotal = DescTotal + BasicTypeTotal; - -//! The constant array that describes different metadata properties -extern const MetaDataConst ArgState[ArgStateTotal]; - -extern const DataTypeConst DataType[]; - -extern const uint DataTypeTotal; - -// Forward declaration -class Program; -class NullProgram; - -class CalImageReference : public amd::ReferenceCountedObject { - public: - //! Default constructor - CalImageReference(CALimage calImage) : image_(calImage) {} - - //! Get CAL image - CALimage calImage() const { return image_; } - - protected: - //! Default destructor - ~CalImageReference(); - - private: - //! Disable copy constructor - CalImageReference(const CalImageReference&); - - //! Disable operator= - CalImageReference& operator=(const CalImageReference&); - - CALimage image_; //!< CAL kernel image -}; - -//! \class GPU NullKernel - Kernel for offline device -class NullKernel : public device::Kernel { - public: - typedef std::vector arguments_t; - - static constexpr uint UavIdUndefined = 0xffff; - - enum Flags { - LimitWorkgroup = 1 << 0, //!< Limits the workgroup size - PrintfOutput = 1 << 1, //!< Kernel has printf output - PrivateFixed = 1 << 2, //!< Kernel has printf output - ABI64bit = 1 << 3, //!< Kernel has 64 bit ABI - Unused0 = 1 << 4, //!< Unused - Unused1 = 1 << 5, //!< Unused - ImageEnable = 1 << 6, //!< Kernel uses images - ImageWrite = 1 << 7, //!< Kernel writes images - }; - - //! \enum Resource type for binding - enum ResourceType { - Undefined = 0x00000000, //!< resource type will be detected - ConstantBuffer = 0x00000001, //!< resource is a constant buffer - GlobalBuffer = 0x00000002, //!< resource is a global buffer - ArgumentHeapBuffer = 0x00000004, //!< resource is an argument buffer - ArgumentBuffer = 0x00000005, //!< resource is an argument buffer - ArgumentImageRead = 0x00000006, //!< resource is an argument image read - ArgumentImageWrite = 0x00000007, //!< resource is an argument image write - ArgumentConstBuffer = 0x00000008, //!< resource is an argument const buffer - ArgumentCounter = 0x00000009, //!< resource is a global counter - ArgumentUavID = 0x0000000a, //!< resource is a dummy ID read - ArgumentCbID = 0x0000000b, //!< resource is a constant buffer - ArgumentPrintfID = 0x0000000c, //!< resource is a printf buffer - }; - - //! GPU kernel constructor - NullKernel(const std::string& name, //!< The kernel's name - const NullDevice& gpuNullDev, //!< GPU device object - const NullProgram& nullProg //!< Reference to the program - ); - - virtual ~NullKernel(); - - /*! \brief Creates a GPU kernel in CAL - * - * \return True if we successfully created a kernel in CAL - */ - bool create(const std::string& code, //!< IL source code - const std::string& metadata, //!< the kernel metadata structure - const void* binaryCode = NULL, //!< binary machine code for CAL - size_t binarySize = 0 //!< the machine code size - ); - - //! Returns CAL function descriptor - CALimage calImage() const { return calRef_->calImage(); } - - //! Returns TRUE if we successfully retrieved the binary from CAL - bool getCalBinary(void* binary, //!< ISA binary code - size_t size //!< ISA binary size - ) const; - - //! Returns CAL image size - size_t getCalBinarySize() const; - - //! Returns GPU device object, associated with this kernel - const NullDevice& nullDev() const { return gpuDev_; } - - //! Returns GPU device object, associated with this kernel - const NullProgram& nullProg() const { return reinterpret_cast(prog_); } - - //! Returns the kernel's build error - const int32_t buildError() const { return buildError_; } - - //! Returns the kernel's flags - uint flags() const { return flags_; } - - //! Returns TRUE if ABI is for 64 bits - bool abi64Bit() const { return (flags_ & ABI64bit) ? true : false; } - - //! Returns the total number of all arguments - size_t argSize() const { return arguments_.size(); } - - //! Returns instruction count of the current kernel - uint instructionCnt() const { return instructionCnt_; } - - protected: - /*! \brief Parses the metadata structure for the kernel, - * provided by the OpenCL compiler - * - * \return True if we succefully parsed all arguments - */ - bool parseArguments(const std::string& metaData, //!< the program for parsing - uint* uavRefCount //!< an array of reference counters for used UAVs - ); - - //! Returns the argument for the specified index - const KernelArg* argument(uint idx) const { return arguments_[idx]; } - - //! Adds the kernel argument into the list - void addArgument(KernelArg* arg) { arguments_.push_back(arg); } - - //! Returns the argument for the specified sampler's index - const KernelArg* sampler(uint idx) const { return intSamplers_[idx]; } - - //! Returns the total number of all internal samplers - size_t samplerSize() const { return intSamplers_.size(); } - - //! Adds the kernel sampler into the sampler's list - void addSampler(KernelArg* arg) { intSamplers_.push_back(arg); } - - //! Returns UAV raw index for this kernel - uint uavRaw() const { return uavRaw_; } - - int32_t buildError_; //!< Kernel's build error - std::string ilSource_; //!< IL source code of this kernel - - const NullDevice& gpuDev_; //!< GPU device object - - CalImageReference* calRef_; //!< CAL image reference for this kernel - bool internal_; //!< Runtime internal ker - - uint flags_; //!< kernel object flags - arguments_t arguments_; //!< kernel arguments for the execution - arguments_t intSamplers_; //!< predefined intenal kernel samplers - - size_t* cbSizes_; //!< real constant buffer sizes for this kernel - uint numCb_; //!< total number of constant buffers - - uint uavRaw_; //!< UAV used for RAW access - - bool rwAttributes_; //!< backend provides RW attributes for arguments - - uint instructionCnt_; //!< Instruction count - - uint cbId_; //!< UAV used for constant buffer access - uint printfId_; //!< UAV used for printf buffer access - - private: - //! Disable copy constructor - NullKernel(const NullKernel&); - - //! Disable operator= - NullKernel& operator=(const NullKernel&); - - //! Creates a filename for ISA/IL dumps - std::string mkDumpName(const char* extension //!< File extension to append - ) const; - - bool createMultiBinary(uint* imageSize, //!< Multibinary image size - void** image, //!< Multibinary image - const void* isa //!< Kernel HW info - ); - - //! SI HW specific setup for kernels - bool siCreateHwInfo(const void* shader, //!< HW info shader - AMUabiAddEncoding& encoding //!< ABI encoding structure - ); - - //! r800 HW specific setup for kernels - bool r800CreateHwInfo(const void* shader, //!< HW info shader - AMUabiAddEncoding& encoding //!< ABI encoding structure - ); -}; - -//! \class GPU kernel -class Kernel : public NullKernel { - public: - struct InitData { - uint privateSize_; //!< Private ring initial size - uint localSize_; //!< Local ring initial size - uint hwPrivateSize_; //!< HW private ring initial size - uint hwLocalSize_; //!< HW local ring initial size - uint flags_; //!< Kernel initialization flags - }; - - //! GPU kernel constructor - Kernel(const std::string& name, //!< The kernel's name - const Device& gpuDev, //!< GPU device object - const Program& prog, //!< Reference to the program - const InitData* initData_ //!< Initialization data - ); - - //! GPU kernel destructor - virtual ~Kernel(); - - /*! \brief Creates a GPU kernel in CAL - * - * \return True if we successfully created a kernel in CAL - */ - bool create(const std::string& code, //!< IL source code - const std::string& metadata, //!< the kernel metadata structure - const void* binaryCode = NULL, //!< binary machine code for CAL - size_t binarySize = 0 //!< the machine code size - ); - - //! Initializes the CAL program grid for the kernel execution - void setupProgramGrid(VirtualGPU& gpu, //!< virtual GPU device object - size_t workDim, //!< work dimension - const amd::NDRange& glbWorkOffset, //!< global work offset - const amd::NDRange& gblWorkSize, //!< global work size - amd::NDRange& lclWorkSize, //!< local work size - const amd::NDRange& groupOffset, //!< group offsets - const amd::NDRange& glbWorkOffsetOrg, - const amd::NDRange& glbWorkSizeOrg //!< original global work size - ) const; - - /*! \brief Detects if runtime has to disable cache optimization and - * recompiles the kernel - * - * \return True if aliases were detected in the kernel arguments - */ - void processMemObjects(VirtualGPU& gpu, //!< Virtual GPU objects - queue - const amd::Kernel& kernel, //!< AMD kernel object for execution - const_address params, //!< pointer to the param's store - bool nativeMem //!< Native memory objects - ) const; - - /*! \brief Loads all kernel arguments, so we could run the kernel in HW. - * This includes CB update and resource binding - * - * \return True if we succefully loaded the arguments - */ - bool loadParameters(VirtualGPU& gpu, //!< virtual GPU device object - const amd::Kernel& kernel, //!< AMD kernel object for execution - const_address params, //!< pointer to the param's store - bool nativeMem //!< Native memory objects - ) const; - - //! Binds the constant buffers associated with the kernel - bool bindConstantBuffers(VirtualGPU& gpu) const; - - /*! \brief Runs the kernel on HW - * - * \return True if we succefully executed the kernel - */ - bool run(VirtualGPU& gpu, //!< virtual GPU device object - GpuEvent* gpuEvent, //!< Pointer to the GPU event - bool lastRun, //!< Last run in the split execution - bool lastDoppCmd, //!< for last dopp submission kernel dispatch - bool pfpaDoppCmd //!< for PFPA dopp submission kernel dispatch - ) const; - - //! Help function to debug the kernel output - void debug(VirtualGPU& gpu //!< virtual GPU device object - ) const; - - //! Programs internal samplers defined inside the kernel - bool setInternalSamplers(VirtualGPU& gpu //!< Virtual GPU device object - ) const; - - //! Returns TRUE if we successfully retrieved the binary from CAL - bool getCalBinary(void* binary, //!< ISA binary code - size_t size //!< ISA binary size - ) const; - - //! Returns CAL image size - size_t getCalBinarySize() const; - - //! Returns GPU device object, associated with this kernel - const Device& dev() const; - - //! Returns GPU device object, associated with this kernel - const Program& prog() const; - - //! Binds global HW constant buffers - bool bindGlobalHwCb(VirtualGPU& gpu, //!< Virtual GPU device object - VirtualGPU::GslKernelDesc* desc //!< Kernel descriptor - ) const; - - protected: - //! Initializes the kernel parameters for the abstraction layer - bool initParameters(); - - /*! \brief Creates constant buffer resources, associated with the kernel - * - * \return TRUE if we succefully created constant buffers - */ - bool initConstBuffers(); - - private: - //! Disable copy constructor - Kernel(const Kernel&); - - //! Disable operator= - Kernel& operator=(const Kernel&); - - //! \enum Fixed Metadata offsets - enum MetadataOffsets { - GlobalWorkitemOffset = 0, - LocalWorkitemOffset = 1, - GroupsOffset = 2, - PrivateRingOffset = 3, - LocalRingOffset = 4, - MathLibOffset = 5, - GlobalWorkOffsetOffset = 6, - GroupWorkOffsetOffset = 7, - GlobalDataStoreOffset = 8, - DebugOffset = 8, - NDRangeGlobalWorkOffsetOffset = 9, - - // The total number of constants reserved for ABI - TotalABIVectors - }; - - /*! \brief Sets the kernel argument - * - * \return True if we succefully updated the arguments - */ - bool setArgument(VirtualGPU& gpu, //!< Virtual GPU device object - const amd::Kernel& kernel, //!< AMD kernel object - uint idx, //!< the argument index - const_address params,//!< the arguments data - const amd::KernelParameterDescriptor& desc, //!< Argument's descriptor - bool nativeMem //!< Native memory objects - ) const; - - /*! \brief Initializes local and private buffer ranges - * - * \return True if we succefully initialized the ranges - */ - bool initLocalPrivateRanges(VirtualGPU& gpu //!< Virtual GPU device object - ) const; - - //! Sets local and private buffer ranges - void setLocalPrivateRanges(VirtualGPU& gpu //!< Virtual GPU device object - ) const; - - //! Sets the sampler's parameters for the image look-up - void setSampler(VirtualGPU& gpu, //!< virtual GPU device object - uint32_t state, //!< sampler state - uint physUnit //!< sampler's number - ) const; - - /*! \brief Binds resource - * - * \return True if we succefully created constant buffers - */ - bool bindResource(VirtualGPU& gpu, //!< virtual GPU device object - const Memory& memory, //!< memory for binding - uint paramIdx, //!< index of the parameter - ResourceType type, //!< resource type - uint physUnit, //!< PhysUnit - size_t offset = 0) const; - - //! Unbinds all resources for the kernel - void unbindResources(VirtualGPU& gpu, //!< virtual GPU device object - GpuEvent gpuEvent, //!< GPU event that will be associated with the resources - bool lastRun //!< last run in the split execution - ) const; - - //! Copies image constants to the constant buffer - void copyImageConstants(const amd::Image* amdImage, //!< Abstraction layer image object - ImageConstants* imageData //!< Pointer in CB to the image constants - ) const; - - //! Finds local workgroup size - void findLocalWorkSize(size_t workDim, //!< Work dimension - const amd::NDRange& gblWorkSize, //!< Global work size - amd::NDRange& lclWorkSize //!< Local work size - ) const; - - uint hwPrivateSize_; //!< initial HW private size - uint hwLocalSize_; //!< initial HW local size -}; - -enum HSAIL_ADDRESS_QUALIFIER { - HSAIL_ADDRESS_ERROR = 0, - HSAIL_ADDRESS_GLOBAL, - HSAIL_ADDRESS_LOCAL, - HSAIL_MAX_ADDRESS_QUALIFIERS -}; - -enum HSAIL_ARG_TYPE { - HSAIL_ARGTYPE_ERROR = 0, - HSAIL_ARGTYPE_POINTER, - HSAIL_ARGTYPE_VALUE, - HSAIL_ARGTYPE_IMAGE, - HSAIL_ARGTYPE_SAMPLER, - HSAIL_ARGTYPE_QUEUE, - HSAIL_ARGMAX_ARG_TYPES -}; - -enum HSAIL_DATA_TYPE { - HSAIL_DATATYPE_ERROR = 0, - HSAIL_DATATYPE_B1, - HSAIL_DATATYPE_B8, - HSAIL_DATATYPE_B16, - HSAIL_DATATYPE_B32, - HSAIL_DATATYPE_B64, - HSAIL_DATATYPE_S8, - HSAIL_DATATYPE_S16, - HSAIL_DATATYPE_S32, - HSAIL_DATATYPE_S64, - HSAIL_DATATYPE_U8, - HSAIL_DATATYPE_U16, - HSAIL_DATATYPE_U32, - HSAIL_DATATYPE_U64, - HSAIL_DATATYPE_F16, - HSAIL_DATATYPE_F32, - HSAIL_DATATYPE_F64, - HSAIL_DATATYPE_STRUCT, - HSAIL_DATATYPE_OPAQUE, - HSAIL_DATATYPE_MAX_TYPES -}; - -enum HSAIL_ACCESS_TYPE { - HSAIL_ACCESS_TYPE_NONE = 0, - HSAIL_ACCESS_TYPE_RO, - HSAIL_ACCESS_TYPE_WO, - HSAIL_ACCESS_TYPE_RW -}; - -class HSAILKernel : public device::Kernel { - public: - struct Argument { - std::string name_; //!< Argument's name - std::string typeName_; //!< Argument's type name - uint size_; //!< Size in bytes - uint offset_; //!< Argument's offset - uint alignment_; //!< Argument's alignment - HSAIL_ARG_TYPE type_; //!< Type of the argument - HSAIL_ADDRESS_QUALIFIER addrQual_; //!< Address qualifier of the argument - HSAIL_DATA_TYPE dataType_; //!< The type of data - uint numElem_; //!< Number of elements - HSAIL_ACCESS_TYPE access_; //!< Access type for the argument - }; - - // Max number of possible extra (hidden) kernel arguments - static constexpr uint MaxExtraArgumentsNum = 6; - - HSAILKernel(std::string name, HSAILProgram* prog, std::string compileOptions, uint extraArgsNum); - - virtual ~HSAILKernel(); - - //! Initializes the metadata required for this kernel, - //! finalizes the kernel if needed - bool init(amd::hsa::loader::Symbol* sym, bool finalize = false); - - //! Returns a pointer to the hsail argument - const Argument* argument(size_t i) const { return arguments_[i]; } - - //! Returns the number of hsail arguments - size_t numArguments() const { return arguments_.size(); } - - //! Returns GPU device object, associated with this kernel - const Device& dev() const; - - //! Returns HSA program associated with this kernel - const HSAILProgram& prog() const; - - //! Returns LDS size used in this kernel - uint32_t ldsSize() const { return cpuAqlCode_->workgroup_group_segment_byte_size; } - - //! Returns pointer on CPU to AQL code info - const void* cpuAqlCode() const { return cpuAqlCode_; } - - //! Returns memory object with AQL code - gpu::Memory* gpuAqlCode() const { return code_; } - - //! Returns size of AQL code - size_t aqlCodeSize() const { return codeSize_; } - - //! Returns the size of argument buffer - size_t argsBufferSize() const { return cpuAqlCode_->kernarg_segment_byte_size; } - - //! Returns spill reg size per workitem - int spillSegSize() const { return cpuAqlCode_->workitem_private_segment_byte_size; } - - //! Returns AQL packet in CPU memory - //! if the kerenl arguments were successfully loaded, otherwise NULL - hsa_kernel_dispatch_packet_t* loadArguments( - VirtualGPU& gpu, //!< Running GPU context - const amd::Kernel& kernel, //!< AMD kernel object - const amd::NDRangeContainer& sizes, //!< NDrange container - const_address parameters, //!< Application arguments for the kernel - bool nativeMem, //!< Native memory objectes are passed - uint64_t vmDefQueue, //!< GPU VM default queue pointer - uint64_t* vmParentWrap, //!< GPU VM parent aql wrap object - std::vector& memList //!< Memory list for GSL/VidMM handles - ) const; - - //! Returns the kernel index in the program - uint index() const { return index_; } - - //! Returns kernel's extra argument count - uint extraArgumentsNum() const { return extraArgumentsNum_; } - - private: - //! Disable copy constructor - HSAILKernel(const HSAILKernel&); - - //! Disable operator= - HSAILKernel& operator=(const HSAILKernel&); - - //! Creates AQL kernel HW info - bool aqlCreateHWInfo(amd::hsa::loader::Symbol* sym); - - //! Initializes arguments_ and the abstraction layer kernel parameters - void initArgList(const aclArgData* aclArg //!< List of ACL arguments - ); - - //! Initializes Hsail Argument metadata and info - void initHsailArgs(const aclArgData* aclArg //!< List of ACL arguments - ); - - std::vector arguments_; //!< Vector list of HSAIL Arguments - std::string compileOptions_; //!< compile used for finalizing this kernel - amd_kernel_code_t* cpuAqlCode_; //!< AQL kernel code on CPU - uint index_; //!< Kernel index in the program - - gpu::Memory* code_; //!< Memory object with ISA code - size_t codeSize_; //!< Size of ISA code - - char* hwMetaData_; //!< SI metadata - - uint extraArgumentsNum_; //! Number of extra (hidden) kernel arguments -}; - -/*@}*/} // namespace gpu - -#endif /*GPUKERNEL_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpumemory.cpp b/projects/clr/rocclr/device/gpu/gpumemory.cpp deleted file mode 100644 index 46c044f97d..0000000000 --- a/projects/clr/rocclr/device/gpu/gpumemory.cpp +++ /dev/null @@ -1,1141 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -//! Implementation of GPU device memory management - -#include "top.hpp" -#include "thread/thread.hpp" -#include "thread/monitor.hpp" -#include "device/device.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpublit.hpp" - -#ifdef _WIN32 -#include -#include "amdocl/cl_d3d9_amd.hpp" -#include "amdocl/cl_d3d10_amd.hpp" -#include "amdocl/cl_d3d11_amd.hpp" -#endif //_WIN32 -#include "amdocl/cl_gl_amd.hpp" - -#include -#include -#include -#include - -//! Turn this on to enable sanity checks before and after every heap operation. -#if DEBUG -#define EXTRA_HEAP_CHECKS 1 -#endif // DEBUG - -namespace gpu { - -Memory::Memory(const Device& gpuDev, amd::Memory& owner, size_t size) - : device::Memory(owner), - Resource(gpuDev, size / Device::Heap::ElementSize, Device::Heap::ElementType) { - init(); - - if (owner.parent() != NULL) { - flags_ |= SubMemoryObject; - } -} - -Memory::Memory(const Device& gpuDev, size_t size) - : device::Memory(size), - Resource(gpuDev, amd::alignUp(size, Device::Heap::ElementSize) / Device::Heap::ElementSize, - Device::Heap::ElementType) { - init(); -} - -Memory::Memory(const Device& gpuDev, amd::Memory& owner, size_t width, cmSurfFmt format) - : device::Memory(owner), Resource(gpuDev, width, format) { - init(); - - if (owner.parent() != NULL) { - flags_ |= SubMemoryObject; - } -} - -Memory::Memory(const Device& gpuDev, size_t size, size_t width, cmSurfFmt format) - : device::Memory(size), Resource(gpuDev, width, format) { - init(); -} - -Memory::Memory(const Device& gpuDev, amd::Memory& owner, size_t width, size_t height, size_t depth, - cmSurfFmt format, gslChannelOrder chOrder, cl_mem_object_type imageType, - uint mipLevels) - : device::Memory(owner), - Resource(gpuDev, width, height, depth, format, chOrder, imageType, mipLevels) { - init(); - - if (owner.parent() != NULL) { - flags_ |= SubMemoryObject; - } -} - -Memory::Memory(const Device& gpuDev, size_t size, size_t width, size_t height, size_t depth, - cmSurfFmt format, gslChannelOrder chOrder, cl_mem_object_type imageType, - uint mipLevels) - : device::Memory(size), - Resource(gpuDev, width, height, depth, format, chOrder, imageType, mipLevels) { - init(); -} - -void Memory::init() { - indirectMapCount_ = 0; - interopType_ = InteropNone; - interopMemory_ = NULL; - pinnedMemory_ = NULL; - parent_ = NULL; -} - -#ifdef _WIN32 -static HANDLE getSharedHandle(IUnknown* pIface) { - // Sanity checks - assert(pIface != NULL); - - HRESULT hRes; - HANDLE hShared; - IDXGIResource* pDxgiRes = NULL; - if ((hRes = (const_cast(pIface)) - ->QueryInterface(__uuidof(IDXGIResource), (void**)&pDxgiRes)) != S_OK) { - return (HANDLE)0; - } - if (!pDxgiRes) { - return (HANDLE)0; - } - hRes = pDxgiRes->GetSharedHandle(&hShared); - pDxgiRes->Release(); - if (hRes != S_OK) { - return (HANDLE)0; - } - return hShared; -} -#endif //_WIN32 - -bool Memory::create(Resource::MemoryType memType, Resource::CreateParams* params) { - bool result; - - // Reset the flag in case we reallocate the heap in local/remote - flags_ &= ~HostMemoryDirectAccess; - - // Create a resource in CAL - result = Resource::create(memType, params); - - // Check if CAL created a resource - if (result) { - switch (memoryType()) { - case Resource::Pinned: - case Resource::ExternalPhysical: - // Marks memory object for direct GPU access to the host memory - flags_ |= HostMemoryDirectAccess; - break; - case Resource::Remote: - case Resource::RemoteUSWC: - if (!cal()->tiled_) { - // Marks memory object for direct GPU access to the host memory - flags_ |= HostMemoryDirectAccess; - } - break; - case Resource::View: { - Resource::ViewParams* view = reinterpret_cast(params); - if (view->resource_->memoryType() == Resource::Persistent) { - flags_ |= HostMemoryDirectAccess; - } - // Check if parent was allocated in system memory - if ((view->resource_->memoryType() == Resource::Pinned) || - (((view->resource_->memoryType() == Resource::Remote) || - (view->resource_->memoryType() == Resource::RemoteUSWC)) && - // @todo Enable unconditional optimization for remote memory - // Check for external allocation, to avoid the optimization - // for non-VM (double copy) mode - (owner() != NULL) && - ((owner()->getMemFlags() & CL_MEM_ALLOC_HOST_PTR) || dev().settings().remoteAlloc_))) { - // Marks memory object for direct GPU access to the host memory - flags_ |= HostMemoryDirectAccess; - } - if ((view->owner_ != NULL) && (view->owner_->parent() != NULL)) { - parent_ = reinterpret_cast(view->memory_); - flags_ |= SubMemoryObject; - } - break; - } - case Resource::ImageView: { - Resource::ImageViewParams* view = reinterpret_cast(params); - parent_ = reinterpret_cast(view->memory_); - flags_ |= SubMemoryObject | (parent_->flags_ & HostMemoryDirectAccess); - break; - } - case Resource::ImageBuffer: { - Resource::ImageBufferParams* view = reinterpret_cast(params); - parent_ = reinterpret_cast(view->memory_); - flags_ |= SubMemoryObject | (parent_->flags_ & HostMemoryDirectAccess); - break; - } - default: - break; - } - } - - return result; -} - -bool Memory::processGLResource(GLResourceOP operation) { - bool retVal = false; - switch (operation) { - case GLDecompressResource: - retVal = gslGLAcquire(); - break; - case GLInvalidateFBO: - retVal = gslGLRelease(); - break; - default: - assert(false && "unknown GLResourceOP"); - } - return retVal; -} - - -bool Memory::createInterop(InteropType type) { - Resource::MemoryType memType = Resource::Empty; - Resource::OGLInteropParams oglRes; -#ifdef _WIN32 - Resource::D3DInteropParams d3dRes; -#endif //_WIN32 - - // Only external objects support interop - assert(owner() != NULL); - - Resource::CreateParams* createParams = NULL; - - amd::InteropObject* interop = owner()->getInteropObj(); - assert((interop != NULL) && "An invalid interop object is impossible!"); - - amd::GLObject* glObject = interop->asGLObject(); - -#ifdef _WIN32 - amd::D3D10Object* d3d10Object = interop->asD3D10Object(); - amd::D3D11Object* d3d11Object = interop->asD3D11Object(); - amd::D3D9Object* d3d9Object = interop->asD3D9Object(); - - if (d3d10Object != NULL) { - createParams = &d3dRes; - - d3dRes.owner_ = owner(); - - const amd::D3D10ObjDesc_t* objDesc = d3d10Object->getObjDesc(); - - memType = Resource::D3D10Interop; - - // Get shared handle - if ((d3dRes.handle_ = getSharedHandle(d3d10Object->getD3D10Resource()))) { - d3dRes.iDirect3D_ = static_cast(d3d10Object->getD3D10Resource()); - d3dRes.type_ = Resource::InteropTypeless; - } - - d3dRes.misc = 0; - // Find D3D10 object type - switch (objDesc->objDim_) { - case D3D10_RESOURCE_DIMENSION_BUFFER: - d3dRes.type_ = Resource::InteropVertexBuffer; - break; - case D3D10_RESOURCE_DIMENSION_TEXTURE1D: - case D3D10_RESOURCE_DIMENSION_TEXTURE2D: - case D3D10_RESOURCE_DIMENSION_TEXTURE3D: - d3dRes.type_ = Resource::InteropTexture; - if (objDesc->mipLevels_ > 1) { - d3dRes.type_ = Resource::InteropTextureViewLevel; - - if (objDesc->arraySize_ > 1) { - d3dRes.layer_ = d3d10Object->getSubresource() / objDesc->mipLevels_; - d3dRes.mipLevel_ = d3d10Object->getSubresource() % objDesc->mipLevels_; - } else { - d3dRes.layer_ = 0; - d3dRes.mipLevel_ = d3d10Object->getSubresource(); - } - } - break; - default: - return false; - break; - } - } else if (d3d11Object != NULL) { - createParams = &d3dRes; - - d3dRes.owner_ = owner(); - - const amd::D3D11ObjDesc_t* objDesc = d3d11Object->getObjDesc(); - - memType = Resource::D3D11Interop; - - // Get shared handle - if ((d3dRes.handle_ = getSharedHandle(d3d11Object->getD3D11Resource()))) { - d3dRes.iDirect3D_ = static_cast(d3d11Object->getD3D11Resource()); - d3dRes.type_ = Resource::InteropTypeless; - } - - d3dRes.misc = 0; - // Find D3D11 object type - switch (objDesc->objDim_) { - case D3D11_RESOURCE_DIMENSION_BUFFER: - d3dRes.type_ = Resource::InteropVertexBuffer; - break; - case D3D11_RESOURCE_DIMENSION_TEXTURE1D: - case D3D11_RESOURCE_DIMENSION_TEXTURE2D: - case D3D11_RESOURCE_DIMENSION_TEXTURE3D: - d3dRes.type_ = Resource::InteropTexture; - d3dRes.layer_ = d3d11Object->getPlane(); - d3dRes.misc = d3d11Object->getMiscFlag(); - if (objDesc->mipLevels_ > 1) { - d3dRes.type_ = Resource::InteropTextureViewLevel; - - if (objDesc->arraySize_ > 1) { - d3dRes.layer_ = d3d11Object->getSubresource() / objDesc->mipLevels_; - d3dRes.mipLevel_ = d3d11Object->getSubresource() % objDesc->mipLevels_; - } else { - d3dRes.layer_ = 0; - d3dRes.mipLevel_ = d3d11Object->getSubresource(); - } - } - break; - default: - return false; - break; - } - } else if (d3d9Object != NULL) { - createParams = &d3dRes; - - d3dRes.owner_ = owner(); - - const amd::D3D9ObjDesc_t* objDesc = d3d9Object->getObjDesc(); - - memType = Resource::D3D9Interop; - - // Get shared handle - if ((d3dRes.handle_ = d3d9Object->getD3D9SharedHandle())) { - d3dRes.iDirect3D_ = static_cast(d3d9Object->getD3D9Resource()); - d3dRes.type_ = Resource::InteropSurface; - d3dRes.mipLevel_ = 0; - d3dRes.layer_ = d3d9Object->getPlane(); - d3dRes.misc = d3d9Object->getMiscFlag(); - } - } else -#endif //_WIN32 - if (glObject != NULL) { - createParams = &oglRes; - - oglRes.owner_ = owner(); - - memType = Resource::OGLInterop; - - // Fill the interop creation parameters - oglRes.handle_ = static_cast(glObject->getGLName()); - - // Find OGL object type - switch (glObject->getCLGLObjectType()) { - case CL_GL_OBJECT_BUFFER: - oglRes.type_ = Resource::InteropVertexBuffer; - break; - case CL_GL_OBJECT_TEXTURE_BUFFER: - case CL_GL_OBJECT_TEXTURE1D: - case CL_GL_OBJECT_TEXTURE1D_ARRAY: - case CL_GL_OBJECT_TEXTURE2D: - case CL_GL_OBJECT_TEXTURE2D_ARRAY: - case CL_GL_OBJECT_TEXTURE3D: - oglRes.type_ = Resource::InteropTexture; - if (GL_TEXTURE_CUBE_MAP == glObject->getGLTarget()) { - switch (glObject->getCubemapFace()) { - case GL_TEXTURE_CUBE_MAP_POSITIVE_X: - case GL_TEXTURE_CUBE_MAP_NEGATIVE_X: - case GL_TEXTURE_CUBE_MAP_POSITIVE_Y: - case GL_TEXTURE_CUBE_MAP_NEGATIVE_Y: - case GL_TEXTURE_CUBE_MAP_POSITIVE_Z: - case GL_TEXTURE_CUBE_MAP_NEGATIVE_Z: - oglRes.type_ = Resource::InteropTextureViewCube; - oglRes.layer_ = glObject->getCubemapFace() - GL_TEXTURE_CUBE_MAP_POSITIVE_X; - oglRes.mipLevel_ = glObject->getGLMipLevel(); - break; - default: - break; - } - } else if (glObject->getGLMipLevel() != 0) { - oglRes.type_ = Resource::InteropTextureViewLevel; - oglRes.layer_ = 0; - oglRes.mipLevel_ = glObject->getGLMipLevel(); - } - break; - case CL_GL_OBJECT_RENDERBUFFER: - oglRes.type_ = Resource::InteropRenderBuffer; - break; - default: - return false; - break; - } - - oglRes.glPlatformContext_ = owner()->getContext().info().hCtx_; - oglRes.glDeviceContext_ = - owner()->getContext().info().hDev_[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx]; - // We dont pass any flags here for the GL Resource. - oglRes.flags_ = 0; - } else { - return false; - } - - // Get the interop settings - if (type == InteropDirectAccess) { - // Create memory object - if (!create(memType, createParams)) { - return false; - } - } else { - // Allocate Resource object for interop as buffer - interopMemory_ = new Memory( - dev(), size(), amd::alignUp(size(), Device::Heap::ElementSize) / Device::Heap::ElementSize, - Device::Heap::ElementType); - - // Create the interop object in CAL - if (NULL == interopMemory_ || !interopMemory_->create(memType, createParams)) { - delete interopMemory_; - interopMemory_ = NULL; - return false; - } - } - - setInteropType(type); - - return true; -} - -Memory::~Memory() { - // Clean VA cache - dev().removeVACache(this); - - delete interopMemory_; - - // Release associated map target, if any - if (NULL != mapMemory_) { - if (owner()->getSvmPtr() != nullptr) { - owner()->uncommitSvmMemory(); - } - - mapMemory()->unmap(NULL); - mapMemory_->release(); - } - - // Destory pinned memory - if (flags_ & PinnedMemoryAlloced) { - delete pinnedMemory_; - } - - if ((owner() != NULL) && isHostMemDirectAccess() && !(flags_ & SubMemoryObject) && - (memoryType() != Resource::ExternalPhysical)) { - // Unmap memory if direct access was requested - unmap(NULL); - } -} - -void Memory::syncCacheFromHost(VirtualGPU& gpu, device::Memory::SyncFlags syncFlags) { - // If the last writer was another GPU, then make a writeback - if (!isHostMemDirectAccess() && (owner()->getLastWriter() != NULL) && - (&dev() != owner()->getLastWriter())) { - mgpuCacheWriteBack(); - } - - // If host memory doesn't have direct access, then we have to synchronize - if (!isHostMemDirectAccess() && (NULL != owner()->getHostMem())) { - bool hasUpdates = true; - - // Make sure the parent of subbuffer is up to date - if (!syncFlags.skipParent_ && (flags_ & SubMemoryObject)) { - gpu::Memory* gpuMemory = dev().getGpuMemory(owner()->parent()); - - //! \note: Skipping the sync for a view doesn't reflect the parent settings, - //! since a view is a small portion of parent - device::Memory::SyncFlags syncFlagsTmp; - - // Sync parent from a view, so views have to be skipped - syncFlagsTmp.skipViews_ = true; - - // Make sure the parent sync is an unique operation. - // If the app uses multiple subbuffers from multiple queues, - // then the parent sync can be called from multiple threads - amd::ScopedLock lock(owner()->parent()->lockMemoryOps()); - gpuMemory->syncCacheFromHost(gpu, syncFlagsTmp); - //! \note Don't do early exit here, since we still have to sync - //! this view, if the parent sync operation was a NOP. - //! If parent was synchronized, then this view sync will be a NOP - } - - // Is this a NOP? - if ((version_ == owner()->getVersion()) || (&dev() == owner()->getLastWriter())) { - hasUpdates = false; - } - - // Update all available views, since we sync the parent - if ((owner()->subBuffers().size() != 0) && (hasUpdates || !syncFlags.skipViews_)) { - device::Memory::SyncFlags syncFlagsTmp; - - // Sync views from parent, so parent has to be skipped - syncFlagsTmp.skipParent_ = true; - - if (hasUpdates) { - // Parent will be synced so update all views with a skip - syncFlagsTmp.skipEntire_ = true; - } else { - // Passthrough the skip entire flag to the views, since - // any view is a submemory of the parent - syncFlagsTmp.skipEntire_ = syncFlags.skipEntire_; - } - - amd::ScopedLock lock(owner()->lockMemoryOps()); - for (auto& sub : owner()->subBuffers()) { - //! \note Don't allow subbuffer's allocation in the worker thread. - //! It may cause a system lock, because possible resource - //! destruction, heap reallocation or subbuffer allocation - static const bool AllocSubBuffer = false; - device::Memory* devSub = sub->getDeviceMemory(dev(), AllocSubBuffer); - if (NULL != devSub) { - gpu::Memory* gpuSub = reinterpret_cast(devSub); - gpuSub->syncCacheFromHost(gpu, syncFlagsTmp); - } - } - } - - // Make sure we didn't have a NOP, - // because this GPU device was the last writer - if (&dev() != owner()->getLastWriter()) { - // Update the latest version - version_ = owner()->getVersion(); - } - - // Exit if sync is a NOP or sync can be skipped - if (!hasUpdates || syncFlags.skipEntire_) { - return; - } - - bool result = false; - static const bool Entire = true; - amd::Coord3D origin(0, 0, 0); - - // If host memory was pinned then make a transfer - if (flags_ & PinnedMemoryAlloced) { - if (cal()->buffer_) { - amd::Coord3D region(owner()->getSize()); - result = gpu.blitMgr().copyBuffer(*pinnedMemory_, *this, origin, origin, region, Entire); - } else { - amd::Image& image = *static_cast(owner()); - result = gpu.blitMgr().copyBufferToImage(*pinnedMemory_, *this, origin, origin, - image.getRegion(), Entire, image.getRowPitch(), - image.getSlicePitch()); - } - } - - if (!result) { - if (cal()->buffer_) { - amd::Coord3D region(owner()->getSize()); - result = gpu.blitMgr().writeBuffer(owner()->getHostMem(), *this, origin, region, Entire); - } else { - amd::Image& image = *static_cast(owner()); - result = gpu.blitMgr().writeImage(owner()->getHostMem(), *this, origin, image.getRegion(), - image.getRowPitch(), image.getSlicePitch(), Entire); - } - } - - //!@todo A wait isn't really necessary. However - //! Linux no-VM may have extra random failures. - wait(gpu); - - // Should never fail - assert(result && "Memory synchronization failed!"); - } -} - -void Memory::syncHostFromCache(device::Memory::SyncFlags syncFlags) { - // Sanity checks - assert(owner() != NULL); - - // If host memory doesn't have direct access, then we have to synchronize - if (!isHostMemDirectAccess()) { - bool hasUpdates = true; - - // Make sure the parent of subbuffer is up to date - if (!syncFlags.skipParent_ && (flags_ & SubMemoryObject)) { - device::Memory* m = owner()->parent()->getDeviceMemory(dev()); - - //! \note: Skipping the sync for a view doesn't reflect the parent settings, - //! since a view is a small portion of parent - device::Memory::SyncFlags syncFlagsTmp; - - // Sync parent from a view, so views have to be skipped - syncFlagsTmp.skipViews_ = true; - - // Make sure the parent sync is an unique operation. - // If the app uses multiple subbuffers from multiple queues, - // then the parent sync can be called from multiple threads - amd::ScopedLock lock(owner()->parent()->lockMemoryOps()); - m->syncHostFromCache(syncFlagsTmp); - //! \note Don't do early exit here, since we still have to sync - //! this view, if the parent sync operation was a NOP. - //! If parent was synchronized, then this view sync will be a NOP - } - - // Is this a NOP? - if ((NULL == owner()->getLastWriter()) || (version_ == owner()->getVersion())) { - hasUpdates = false; - } - - // Update all available views, since we sync the parent - if ((owner()->subBuffers().size() != 0) && (hasUpdates || !syncFlags.skipViews_)) { - device::Memory::SyncFlags syncFlagsTmp; - - // Sync views from parent, so parent has to be skipped - syncFlagsTmp.skipParent_ = true; - - if (hasUpdates) { - // Parent will be synced so update all views with a skip - syncFlagsTmp.skipEntire_ = true; - } else { - // Passthrough the skip entire flag to the views, since - // any view is a submemory of the parent - syncFlagsTmp.skipEntire_ = syncFlags.skipEntire_; - } - - amd::ScopedLock lock(owner()->lockMemoryOps()); - for (auto& sub : owner()->subBuffers()) { - //! \note Don't allow subbuffer's allocation in the worker thread. - //! It may cause a system lock, because possible resource - //! destruction, heap reallocation or subbuffer allocation - static const bool AllocSubBuffer = false; - device::Memory* devSub = sub->getDeviceMemory(dev(), AllocSubBuffer); - if (NULL != devSub) { - gpu::Memory* gpuSub = reinterpret_cast(devSub); - gpuSub->syncHostFromCache(syncFlagsTmp); - } - } - } - - // Make sure we didn't have a NOP, - // because CPU was the last writer - if (NULL != owner()->getLastWriter()) { - // Mark parent as up to date, set our version accordingly - version_ = owner()->getVersion(); - } - - // Exit if sync is a NOP or sync can be skipped - if (!hasUpdates || syncFlags.skipEntire_) { - return; - } - - bool result = false; - static const bool Entire = true; - amd::Coord3D origin(0, 0, 0); - - // If backing store was pinned then make a transfer - if (flags_ & PinnedMemoryAlloced) { - if (cal()->buffer_) { - amd::Coord3D region(owner()->getSize()); - result = dev().xferMgr().copyBuffer(*this, *pinnedMemory_, origin, origin, region, Entire); - } else { - amd::Image& image = *static_cast(owner()); - result = dev().xferMgr().copyImageToBuffer(*this, *pinnedMemory_, origin, origin, - image.getRegion(), Entire, image.getRowPitch(), - image.getSlicePitch()); - } - } - - // Just do a basic host read - if (!result) { - if (cal()->buffer_) { - amd::Coord3D region(owner()->getSize()); - result = dev().xferMgr().readBuffer(*this, owner()->getHostMem(), origin, region, Entire); - } else { - amd::Image& image = *static_cast(owner()); - result = dev().xferMgr().readImage(*this, owner()->getHostMem(), origin, image.getRegion(), - image.getRowPitch(), image.getSlicePitch(), Entire); - } - } - - // Should never fail - assert(result && "Memory synchronization failed!"); - } -} - -gpu::Memory* Memory::createBufferView(amd::Memory& subBufferOwner) { - gpu::Memory* viewMemory; - Resource::ViewParams params; - - size_t offset = subBufferOwner.getOrigin(); - size_t size = subBufferOwner.getSize(); - - // Create a memory object - viewMemory = new gpu::Memory(dev(), subBufferOwner, size); - if (NULL == viewMemory) { - return NULL; - } - - params.owner_ = &subBufferOwner; - params.gpu_ = static_cast(subBufferOwner.getVirtualDevice()); - params.offset_ = offset; - params.size_ = size; - params.resource_ = this; - params.memory_ = this; - if (!viewMemory->create(Resource::View, ¶ms)) { - delete viewMemory; - return NULL; - } - - // Explicitly set the host memory location, - // because the parent location could change after reallocation - if (NULL != owner()->getHostMem()) { - subBufferOwner.setHostMem(reinterpret_cast(owner()->getHostMem()) + offset); - } else { - subBufferOwner.setHostMem(NULL); - } - - return viewMemory; -} - -void Memory::decIndMapCount() { - // Map/unmap must be serialized - amd::ScopedLock lock(owner()->lockMemoryOps()); - - if (indirectMapCount_ == 0) { - if (!mipMapped()) { - LogError("decIndMapCount() called when indirectMapCount_ already zero"); - } - return; - } - - // Decrement the counter and release indirect map if it's the last op - if (--indirectMapCount_ == 0) { - if (NULL != mapMemory_) { - amd::Memory* memory = mapMemory_; - amd::Memory* empty = NULL; - - // Get GPU memory - Memory* gpuMemory = mapMemory(); - gpuMemory->unmap(NULL); - - if (!dev().addMapTarget(memory)) { - memory->release(); - } - - // Map/unamp is serialized for the same memory object, - // so it's safe to clear the pointer - assert((mapMemory_ != NULL) && "Mapped buffer should be valid"); - mapMemory_ = NULL; - } - } -} - -// Note - must be called by the device under the async lock, so no spinning -// or long pauses allowed in this function. -void* Memory::allocMapTarget(const amd::Coord3D& origin, const amd::Coord3D& region, uint mapFlags, - size_t* rowPitch, size_t* slicePitch) { - // Sanity checks - assert(owner() != NULL); - - // Map/unmap must be serialized - amd::ScopedLock lock(owner()->lockMemoryOps()); - - address mapAddress = NULL; - size_t offset = origin[0]; - - // For SVM implementation, we cannot use cached map. if svm space, use the svm host pointer - void* initHostPtr = owner()->getSvmPtr(); - if (NULL != initHostPtr) { - owner()->commitSvmMemory(); - } - - if (owner()->numDevices() > 1) { - if ((NULL == initHostPtr) && (owner()->getHostMem() == NULL)) { - static const bool forceAllocHostMem = true; - if (!owner()->allocHostMemory(NULL, forceAllocHostMem)) { - return NULL; - } - //! \note Ignore pinning result - // bool ok = pinSystemMemory(owner()->getHostMem(), owner()->getSize()); - } - } - - incIndMapCount(); - // If host memory exists, use it - if ((owner()->getHostMem() != NULL) && isDirectMap()) { - mapAddress = reinterpret_cast
(owner()->getHostMem()); - } - // If resource is a persistent allocation, we can use it directly - else if (isPersistentDirectMap()) { - if (NULL == map(NULL)) { - LogError("Could not map target persistent resource"); - decIndMapCount(); - return NULL; - } - mapAddress = data(); - } - // Otherwise we can use a remote resource: - else { - // Are we in range? - size_t elementCount = cal()->width_; - size_t rSize = elementCount * elementSize(); - if (offset >= rSize || offset + region[0] > rSize) { - LogWarning("Memory::allocMapTarget() - offset/size out of bounds"); - return NULL; - } - - // Allocate a map resource if there isn't any yet - if (indirectMapCount_ == 1) { - const static bool SysMem = true; - bool failed = false; - amd::Memory* memory = NULL; - // Search for a possible indirect resource - cl_mem_flags flag = 0; - bool canBeCached = true; - if (NULL != initHostPtr) { - // make sure the host memory is committed already, or we have a big problem. - assert(owner()->isSvmPtrCommited() && "The host svm memory not committed yet!"); - flag = CL_MEM_USE_HOST_PTR; - canBeCached = false; - } else { - memory = dev().findMapTarget(owner()->getSize()); - } - - if (memory == NULL) { - // for map target of svm buffer , we need use svm host ptr - memory = new (dev().context()) amd::Buffer(dev().context(), flag, owner()->getSize()); - - do { - if ((memory == NULL) || !memory->create(initHostPtr, SysMem)) { - failed = true; - break; - } - memory->setCacheStatus(canBeCached); - - Memory* gpuMemory = reinterpret_cast(memory->getDeviceMemory(dev())); - - // Create, Map and get the base pointer for the resource - if ((gpuMemory == NULL) || (NULL == gpuMemory->map(NULL))) { - failed = true; - break; - } - } while (false); - } - - if (failed) { - if (memory != NULL) { - memory->release(); - } - decIndMapCount(); - LogError("Could not map target resource"); - return NULL; - } - - // Map/unamp is serialized for the same memory object, - // so it's safe to assign the new pointer - assert((mapMemory_ == NULL) && "Mapped buffer can't be valid"); - mapMemory_ = memory; - } else { - // Did the map resource allocation fail? - if (mapMemory_ == NULL) { - LogError("Could not map target resource"); - return NULL; - } - } - mapAddress = mapMemory()->data(); - } - - return mapAddress + offset; -} - -bool Memory::pinSystemMemory(void* hostPtr, size_t size) { - bool result = false; - - // If memory has a direct access already, then skip the host memory pinning - if (isHostMemDirectAccess()) { - return true; - } - - // Check if memory is pinned already - if (flags_ & PinnedMemoryAlloced) { - return true; - } - - // Allocate memory for the pinned object - pinnedMemory_ = new Memory(dev(), size); - - if (pinnedMemory_ == NULL) { - return false; - } - - // Check if it's a view - if (flags_ & SubMemoryObject) { - const gpu::Memory* gpuMemory; - if (owner() != NULL) { - gpuMemory = dev().getGpuMemory(owner()->parent()); - } else { - gpuMemory = parent(); - } - - if (gpuMemory->flags_ & PinnedMemoryAlloced) { - Resource::ViewParams params; - params.owner_ = owner(); - params.offset_ = owner()->getOrigin(); - params.size_ = owner()->getSize(); - params.resource_ = gpuMemory->pinnedMemory_; - params.memory_ = NULL; - result = pinnedMemory_->create(Resource::View, ¶ms); - } - } else { - Resource::PinnedParams params; - // Fill resource creation parameters - params.owner_ = owner(); - params.hostMemRef_ = owner()->getHostMemRef(); - params.size_ = size; - - // Create resource - result = pinnedMemory_->create(Resource::Pinned, ¶ms); - } - - if (!result) { - delete pinnedMemory_; - pinnedMemory_ = NULL; - return false; - } - - flags_ |= PinnedMemoryAlloced; - return true; -} - -void* Memory::cpuMap(device::VirtualDevice& vDev, uint flags, uint startLayer, uint numLayers, - size_t* rowPitch, size_t* slicePitch) { - uint resFlags = 0; - if (flags == Memory::CpuReadOnly) { - resFlags = Resource::ReadOnly; - } else if (flags == Memory::CpuWriteOnly) { - resFlags = Resource::WriteOnly; - } - - void* ptr = map(&static_cast(vDev), resFlags, startLayer, numLayers); - if (!cal()->buffer_) { - *rowPitch = cal()->pitch_ * elementSize(); - *slicePitch = cal()->slice_ * elementSize(); - } - return ptr; -} - -void Memory::cpuUnmap(device::VirtualDevice& vDev) { unmap(&static_cast(vDev)); } - -Memory* Memory::mapMemory() const { - Memory* map = NULL; - if (NULL != mapMemory_) { - map = reinterpret_cast(mapMemory_->getDeviceMemory(dev())); - } - return map; -} - -void Memory::mgpuCacheWriteBack() { - // Lock memory object, so only one write back can occur - amd::ScopedLock lock(owner()->lockMemoryOps()); - - // Attempt to allocate a staging buffer if don't have any - if (owner()->getHostMem() == NULL) { - if (nullptr != owner()->getSvmPtr()) { - owner()->commitSvmMemory(); - owner()->setHostMem(owner()->getSvmPtr()); - } else { - static const bool forceAllocHostMem = true; - owner()->allocHostMemory(nullptr, forceAllocHostMem); - } - } - // Make synchronization - if (owner()->getHostMem() != NULL) { - //! \note Ignore pinning result - bool ok = pinSystemMemory(owner()->getHostMem(), owner()->getSize()); - owner()->cacheWriteBack(); - } -} - -Memory* Buffer::createBufferView(amd::Memory& subBufferOwner) const { - gpu::Memory* subBuffer; - Resource::ViewParams params; - - size_t offset = subBufferOwner.getOrigin(); - size_t size = subBufferOwner.getSize(); - - // Create a memory object - subBuffer = new gpu::Buffer(dev(), subBufferOwner, size); - if (NULL == subBuffer) { - return NULL; - } - - // Allocate a view for this buffer object - params.owner_ = &subBufferOwner; - params.offset_ = offset; - params.size_ = size; - params.resource_ = this; - params.memory_ = this; - - if (!subBuffer->create(Resource::View, ¶ms)) { - delete subBuffer; - return NULL; - } - - return subBuffer; -} - -void* Image::allocMapTarget(const amd::Coord3D& origin, const amd::Coord3D& region, uint mapFlags, - size_t* rowPitch, size_t* slicePitch) { - // Sanity checks - assert(owner() != NULL); - bool useRemoteResource = true; - size_t slicePitchTmp = 0; - size_t height = cal()->height_; - size_t depth = cal()->depth_; - - // Map/unmap must be serialized - amd::ScopedLock lock(owner()->lockMemoryOps()); - - address mapAddress = NULL; - size_t offset = origin[0]; - - incIndMapCount(); - - // If host memory exists, use it - if ((owner()->getHostMem() != NULL) && isDirectMap()) { - useRemoteResource = false; - mapAddress = reinterpret_cast
(owner()->getHostMem()); - amd::Image* amdImage = owner()->asImage(); - - // Calculate the offset in bytes - offset *= elementSize(); - - // Update the row and slice pitches value - *rowPitch = - (amdImage->getRowPitch() == 0) ? (cal()->width_ * elementSize()) : amdImage->getRowPitch(); - slicePitchTmp = - (amdImage->getSlicePitch() == 0) ? (height * (*rowPitch)) : amdImage->getSlicePitch(); - - // Adjust the offset in Y and Z dimensions - offset += origin[1] * (*rowPitch); - offset += origin[2] * slicePitchTmp; - } - // If resource is a persistent allocation, we can use it directly - //! @note Even if resource is a persistent allocation, - //! runtime can't use it directly, - //! because CAL volume map doesn't work properly. - //! @todo arrays can be added for persistent lock with some CAL changes - else if (isPersistentDirectMap()) { - if (NULL == map(NULL)) { - useRemoteResource = true; - LogError("Could not map target persistent resource, try remote resource"); - } else { - useRemoteResource = false; - mapAddress = data(); - - // Calculate the offset in bytes - offset *= elementSize(); - - // Update the row pitch value - *rowPitch = cal()->pitch_ * elementSize(); - - // Adjust the offset in Y dimension - offset += origin[1] * (*rowPitch); - } - } - - // Otherwise we can use a remote resource: - if (useRemoteResource) { - // Calculate X offset in bytes - offset *= elementSize(); - - // Allocate a map resource if there isn't any yet - if (indirectMapCount_ == 1) { - const static bool SysMem = true; - bool failed = false; - amd::Memory* memory; - - // Search for a possible indirect resource - memory = dev().findMapTarget(owner()->getSize()); - - if (memory == NULL) { - // Allocate a new buffer to use as the map target - //! @note Allocate a 1D buffer, since CAL issues with 3D - //! Also HW doesn't support untiled images - memory = new (dev().context()) - amd::Buffer(dev().context(), 0, cal()->width_ * height * depth * elementSize()); - memory->setVirtualDevice(owner()->getVirtualDevice()); - - do { - if ((memory == NULL) || !memory->create(NULL, SysMem)) { - failed = true; - break; - } - - Memory* gpuMemory = reinterpret_cast(memory->getDeviceMemory(dev())); - - // Create, Map and get the base pointer for the resource - if ((gpuMemory == NULL) || (NULL == gpuMemory->map(NULL))) { - failed = true; - break; - } - } while (false); - } - - if (failed) { - if (memory != NULL) { - memory->release(); - } - decIndMapCount(); - LogError("Could not map target resource"); - return NULL; - } - - // Map/unamp is serialized for the same memory object, - // so it's safe to assign the new pointer - assert((mapMemory_ == NULL) && "Mapped buffer can't be valid"); - mapMemory_ = memory; - } else { - // Did the map resource allocation fail? - if (mapMemory_ == NULL) { - LogError("Could not map target resource"); - return NULL; - } - } - - mapAddress = mapMemory()->data(); - - // Update the row and slice pitches value - *rowPitch = region[0] * elementSize(); - if (cal()->dimension_ == GSL_MOA_TEXTURE_1D_ARRAY) { - slicePitchTmp = *rowPitch; - } else { - slicePitchTmp = *rowPitch * region[1]; - } - // Use start of the indirect buffer - offset = 0; - } - - if (slicePitch != NULL) { - *slicePitch = slicePitchTmp; - } - - return mapAddress + offset; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpumemory.hpp b/projects/clr/rocclr/device/gpu/gpumemory.hpp deleted file mode 100644 index adf7f4f3a9..0000000000 --- a/projects/clr/rocclr/device/gpu/gpumemory.hpp +++ /dev/null @@ -1,286 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUMEMORY_HPP_ -#define GPUMEMORY_HPP_ - -#include "top.hpp" -#include "device/gpu/gpuresource.hpp" -#include - -/*! \addtogroup GPU - * @{ - */ -namespace device { -class Memory; -} - -//! GPU Device Implementation -namespace gpu { - -class Device; -class Heap; -class Resource; -class Memory; -class VirtualGPU; - -//! GPU memory object. -// Wrapper that can contain a heap block or an interop buffer/image. -class Memory : public device::Memory, public Resource { - public: - enum InteropType { - InteropNone = 0, //!< None interop memory - InteropHwEmulation = 1, //!< Uses HW emulaiton with calMemCopy - InteropDirectAccess = 2 //!< Uses direct access to the interop surface - }; - - //! Constructor (with owner) - Memory(const Device& gpuDev, amd::Memory& owner, size_t size = 0); - - //! Constructor (nonfat version for local scratch mem use without heap block) - Memory(const Device& gpuDev, size_t size); - - //! Constructor memory for buffer (without global heap allocaton) - Memory(const Device& gpuDev, //!< GPU device object - amd::Memory& owner, //!< Abstraction layer memory object - size_t width, //!< Memory width - cmSurfFmt format //!< CAL format - ); - - //! Constructor memory for buffer (without global heap allocaton) - Memory(const Device& gpuDev, //!< GPU device object - size_t size, //!< Memory object size - size_t width, //!< Memory width - cmSurfFmt format //!< CAL format - ); - - //! Constructor memory for images (without global heap allocaton) - Memory(const Device& gpuDev, //!< GPU device object - amd::Memory& owner, //!< Abstraction layer memory object - size_t width, //!< Allocated memory width - size_t height, //!< Allocated memory height - size_t depth, //!< Allocated memory depth - cmSurfFmt format, //!< Memory format - gslChannelOrder chOrder, //!< Channel order - cl_mem_object_type imageType, //!< CL image type - uint mipLevels //!< The number of mip levels - ); - - //! Constructor memory for images (without global heap allocaton) - Memory(const Device& gpuDev, //!< GPU device object - size_t size, //!< Memory object size - size_t width, //!< Allocated memory width - size_t height, //!< Allocated memory height - size_t depth, //!< Allocated memory depth - cmSurfFmt format, //!< Memory format - gslChannelOrder chOrder, //!< Channel order - cl_mem_object_type imageType, //!< CL image type - uint mipLevels //!< The number of mip levels - ); - - //! Default destructor - ~Memory(); - - //! Creates the interop memory - bool createInterop(InteropType type //!< The interop type - ); - - //! Overloads the resource create method - virtual bool create(Resource::MemoryType memType, //!< Memory type - Resource::CreateParams* params = NULL //!< Prameters for create - ); - - //! Allocate memory for API-level maps - virtual void* allocMapTarget(const amd::Coord3D& origin, //!< The map location in memory - const amd::Coord3D& region, //!< The map region in memory - uint mapFlags, //!< Map flags - size_t* rowPitch = NULL, //!< Row pitch for the mapped memory - size_t* slicePitch = NULL //!< Slice for the mapped memory - ); - - //! Pins system memory associated with this memory object - virtual bool pinSystemMemory(void* hostPtr, //!< System memory address - size_t size //!< Size of allocated system memory - ); - - //! Releases indirect map surface - virtual void releaseIndirectMap() { decIndMapCount(); } - - //! Map the device memory to CPU visible - virtual void* cpuMap(device::VirtualDevice& vDev, //!< Virtual device for map operaiton - uint flags = 0, //!< flags for the map operation - // Optimization for multilayer map/unmap - uint startLayer = 0, //!< Start layer for multilayer map - uint numLayers = 0, //!< End layer for multilayer map - size_t* rowPitch = NULL, //!< Row pitch for the device memory - size_t* slicePitch = NULL //!< Slice pitch for the device memory - ); - - //! Unmap the device memory - virtual void cpuUnmap(device::VirtualDevice& vDev //!< Virtual device for unmap operaiton - ); - - //! Updates device memory from the owner's host allocation - void syncCacheFromHost(VirtualGPU& gpu, //!< Virtual GPU device object - //! Synchronization flags - device::Memory::SyncFlags syncFlags = device::Memory::SyncFlags()); - - //! Updates the owner's host allocation from device memory - virtual void syncHostFromCache( - //! Synchronization flags - device::Memory::SyncFlags syncFlags = device::Memory::SyncFlags()); - - //! Creates a view from current resource - virtual Memory* createBufferView( - amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner - ); - - virtual uint64_t virtualAddress() const override { return (vmAddress() + pinOffset()); } - - //! Allocates host memory for synchronization with MGPU context - void mgpuCacheWriteBack(); - - //! Accessors for indirect map memory object - Memory* mapMemory() const; - - //! Returns the interop memory for this memory object - Memory* interop() const { return interopMemory_; } - - //! Gets interop type for this memory object - InteropType interopType() const { return interopType_; } - - //! Sets interop type for this memory object - void setInteropType(InteropType type) { interopType_ = type; } - - //! Set the owner - void setOwner(amd::Memory* owner) { owner_ = owner; } - - // Decompress GL depth-stencil/MSAA resources for CL access - // Invalidates any FBOs the resource may be bound to, otherwise the GL driver may crash. - virtual bool processGLResource(GLResourceOP operation); - - //! Returns the interop resource for this memory object - const Memory* parent() const { return parent_; } - - //! Returns TRUE if direct map is acceaptable. The method detects - //! forced USWC memory on APU and will cause a switch to - //! indirect map for allocations with a possibility of host read - bool isDirectMap() { - return (isCacheable() || !isHostMemDirectAccess() || - (owner()->getMemFlags() & - (CL_MEM_ALLOC_HOST_PTR | CL_MEM_HOST_WRITE_ONLY | CL_MEM_READ_ONLY))); - } - - protected: - //! Decrement map count - void decIndMapCount(); - - //! Initialize the object members - void init(); - - private: - //! Disable copy constructor - Memory(const Memory&); - - //! Disable operator= - Memory& operator=(const Memory&); - - InteropType interopType_; //!< Interop type - Memory* interopMemory_; //!< interop memory - - Memory* pinnedMemory_; //!< Memory used as pinned system memory - const Memory* parent_; //!< Parent memory object -}; - -class Buffer : public gpu::Memory { - public: - //! Buffer constructor - Buffer(const Device& gpuDev, //!< GPU device object - amd::Memory& owner, //!< Abstraction layer memory object - size_t size //!< Buffer size - ) - : gpu::Memory(gpuDev, owner, amd::alignUp(size, ElementSize) / ElementSize, ElementType) {} - - //! Creates a view from current resource - virtual Memory* createBufferView( - amd::Memory& subBufferOwner //!< The abstraction layer subbuf owner - ) const; - - private: - //! Disable copy constructor - Buffer(const Buffer&); - - //! Disable operator= - Buffer& operator=(const Buffer&); - - //! The size of buffer element in bytes - static constexpr size_t ElementSize = 4; - - //! The type of buffer element - static constexpr cmSurfFmt ElementType = CM_SURF_FMT_R32I; -}; - -class Image : public gpu::Memory { - public: - //! Image constructor - Image(const Device& gpuDev, //!< GPU device object - amd::Memory& owner, //!< Abstraction layer memory object - size_t width, //!< Allocated memory width - size_t height, //!< Allocated memory height - size_t depth, //!< Allocated memory depth - cmSurfFmt format, //!< Memory format - gslChannelOrder chOrder, //!< Channel order - cl_mem_object_type imageType, //!< CL image type - uint mipLevels //!< The number of mip levels - ) - : gpu::Memory(gpuDev, owner, width, height, depth, format, chOrder, imageType, mipLevels) {} - - //! Image constructor - Image(const Device& gpuDev, //!< GPU device object - size_t size, //!< Memory size - size_t width, //!< Allocated memory width - size_t height, //!< Allocated memory height - size_t depth, //!< Allocated memory depth - cmSurfFmt format, //!< Memory format - gslChannelOrder chOrder, //!< Channel order - cl_mem_object_type imageType, //!< CL image type - uint mipLevels //!< The number of mip levels - ) - : gpu::Memory(gpuDev, size, width, height, depth, format, chOrder, imageType, mipLevels) {} - - //! Allocate memory for API-level maps - virtual void* allocMapTarget(const amd::Coord3D& origin, //!< The map location in memory - const amd::Coord3D& region, //!< The map region in memory - uint mapFlags, //!< Map flags - size_t* rowPitch = NULL, //!< Row pitch for the mapped memory - size_t* slicePitch = NULL //!< Slice for the mapped memory - ); - - private: - //! Disable copy constructor - Image(const Image&); - - //! Disable operator= - Image& operator=(const Image&); -}; - -} // namespace gpu - -#endif // GPUMEMORY_HPP_ diff --git a/projects/clr/rocclr/device/gpu/gpuprintf.cpp b/projects/clr/rocclr/device/gpu/gpuprintf.cpp deleted file mode 100644 index e803380c4b..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuprintf.cpp +++ /dev/null @@ -1,657 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "top.hpp" -#include "os/os.hpp" -#include "device/device.hpp" -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpumemory.hpp" -#include "device/gpu/gpukernel.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpuprintf.hpp" -#include -#include -#include - -namespace gpu { - -PrintfDbg::PrintfDbg(Device& device, FILE* file) - : dbgBuffer_(NULL), - dbgFile_(file), - gpuDevice_(device), - wiDbgSize_(0), - initCntValue_(device, 4) {} - -PrintfDbg::~PrintfDbg() { delete dbgBuffer_; } - -bool PrintfDbg::create() { - // Create a resource for the init count value - if (initCntValue_.create(Resource::Remote)) { - uint32_t* value = reinterpret_cast(initCntValue_.map(NULL)); - // The counter starts from 1 - if (NULL != value) { - *value = 1; - } else { - return false; - } - initCntValue_.unmap(NULL); - return true; - } - return false; -} - -bool PrintfDbg::init(VirtualGPU& gpu, bool printfEnabled, const amd::NDRange& size) { - // Set up debug output buffer (if printf active) - if (printfEnabled) { - if (!allocate()) { - return false; - } - - // Make sure that the size isn't bigger than the reported max - if (size.product() <= dev().settings().maxWorkGroupSize_) { - size_t wiDbgSizeTmp; - - // Calculate the debug buffer size per workitem - wiDbgSizeTmp = std::min(dbgBuffer_->size() / size.product(), dev().xferRead().bufSize()); - - // Make sure the size is DWORD aligned - wiDbgSizeTmp = amd::alignDown(wiDbgSizeTmp, sizeof(uint32_t)); - - // If the new size is different, then clear the initial values - if (wiDbgSize_ != wiDbgSizeTmp) { - wiDbgSize_ = wiDbgSizeTmp; - if (!clearWorkitems(gpu, 0, size.product())) { - wiDbgSize_ = 0; - return false; - } - } - } - } - - return true; -} - -bool PrintfDbg::output(VirtualGPU& gpu, bool printfEnabled, const amd::NDRange& size, - const std::vector& printfInfo) { - // Are we expected to generate debug output? - if (printfEnabled && !printfInfo.empty()) { - uint32_t* workitemData; - size_t i, j, k, z; - bool realloc = false; - - // Wait for kernel execution - gpu.waitAllEngines(); - - size_t zdim = 1; - size_t ydim = 1; - size_t xdim = 1; - - switch (size.dimensions()) { - case 3: - zdim = size[2]; - // Fall through ... - case 2: - ydim = size[1]; - // Fall through ... - case 1: - xdim = size[0]; - // Fall through ... - default: - break; - } - - for (k = 0; k < zdim; ++k) { - for (j = 0; j < ydim; ++j) { - for (i = 0; i < xdim; ++i) { - size_t idx = (xdim * (ydim * k + j) + i); - workitemData = mapWorkitem(gpu, idx, &realloc); - - if (NULL != workitemData) { - uint32_t wp = workitemData[0]; // write pointer (i.e. first unwritten element) - // Walk through each PrintfDbg entry - for (z = 1; (z < (wiDbgSize() / sizeof(uint32_t))) && (z < wp);) { - if (printfInfo.size() < workitemData[z]) { - LogError("The format string wasn't reported"); - return false; - } - // Get the PrintfDbg info - const device::PrintfInfo& info = printfInfo[workitemData[z++]]; - // There's something in this buffer - outputDbgBuffer(info, workitemData, z); - } - } - unmapWorkitem(gpu, workitemData); - } - } - } - - // Reallocate debug buffer if necessary - if (!allocate(realloc)) { - return false; - } - } - return true; -} - -uint64_t PrintfDbg::bufOffset() const { return dbgBuffer_->hbOffset(); } - -bool PrintfDbg::allocate(bool realloc) { - if (NULL == dbgBuffer_) { - dbgBuffer_ = dev().createScratchBuffer(dev().info().printfBufferSize_); - } else if (realloc) { - LogWarning("Debug buffer reallocation!"); - // Double the buffer size if it's not big enough - size_t size = dbgBuffer_->size(); - delete dbgBuffer_; - dbgBuffer_ = dev().createScratchBuffer(size << 1); - } - - return (NULL != dbgBuffer_) ? true : false; -} - -bool PrintfDbg::checkFloat(const std::string& fmt) const { - switch (fmt[fmt.size() - 1]) { - case 'e': - case 'E': - case 'f': - case 'g': - case 'G': - case 'a': - return true; - break; - default: - break; - } - return false; -} - -bool PrintfDbg::checkString(const std::string& fmt) const { - if (fmt[fmt.size() - 1] == 's') return true; - return false; -} - -int PrintfDbg::checkVectorSpecifier(const std::string& fmt, size_t startPos, size_t& curPos) const { - int vectorSize = 0; - size_t pos = curPos; - size_t size = curPos - startPos; - - if (size >= 3) { - size = 0; - // no modifiers - if (fmt[curPos - 3] == 'v') { - size = 2; - } - // the modifiers are "h" or "l" - else if (fmt[curPos - 4] == 'v') { - size = 3; - } - // the modifier is "hh" - else if ((curPos >= 5) && (fmt[curPos - 5] == 'v')) { - size = 4; - } - if (size > 0) { - curPos = size; - pos -= curPos; - - // Get vector size - vectorSize = fmt[pos++] - '0'; - // PrintfDbg supports only 2, 3, 4, 8 and 16 wide vectors - switch (vectorSize) { - case 1: - if ((fmt[pos++] - '0') == 6) { - vectorSize = 16; - } else { - vectorSize = 0; - } - break; - case 2: - case 3: - case 4: - case 8: - break; - default: - vectorSize = 0; - break; - } - } - } - - return vectorSize; -} - -static constexpr size_t ConstStr = 0xffffffff; -static constexpr char Separator[] = ",\0"; - -size_t PrintfDbg::outputArgument(const std::string& fmt, bool printFloat, size_t size, - const uint32_t* argument) const { - // Serialize the output to the screen - amd::ScopedLock k(dev().lockAsyncOps()); - - size_t copiedBytes = size; - // Print the string argument, using standard PrintfDbg() - if (checkString(fmt.c_str())) { - // copiedBytes should be as number of printed chars - copiedBytes = 0; - //(null) should be printed - if (*argument == 0) { - amd::Os::printf(fmt.data(), 0); - // copiedBytes = strlen("(null)") - copiedBytes = 6; - } else { - const unsigned char* argumentStr = reinterpret_cast(argument); - amd::Os::printf(fmt.data(), argumentStr); - // copiedBytes = strlen(argumentStr) - while (argumentStr[copiedBytes++] != 0) - ; - } - } - - // Print the argument(except for string ), using standard PrintfDbg() - else { - bool hlModifier = (strstr(fmt.c_str(), "hl") != NULL); - std::string hlFmt; - if (hlModifier) { - hlFmt = fmt; - hlFmt.erase(hlFmt.find_first_of("hl"), 2); - } - switch (size) { - case 0: { - const char* str = reinterpret_cast(argument); - amd::Os::printf(fmt.data(), str); - // Find the string length - while (str[copiedBytes++] != 0) - ; - } break; - case 1: - amd::Os::printf(fmt.data(), *(reinterpret_cast(argument))); - break; - case 2: - case 4: - if (printFloat) { - static const char* fSpecifiers = "eEfgGa"; - std::string fmtF = fmt; - size_t posS = fmtF.find_first_of("%"); - size_t posE = fmtF.find_first_of(fSpecifiers); - if (posS != std::string::npos && posE != std::string::npos) { - fmtF.replace(posS + 1, posE - posS, "s"); - } - float fArg = *(reinterpret_cast(argument)); - float fSign = copysign(1.0, fArg); - if (isinf(fArg) && !isnan(fArg)) { - if (fSign < 0) { - amd::Os::printf(fmtF.data(), "-infinity"); - } else { - amd::Os::printf(fmtF.data(), "infinity"); - } - } else if (isnan(fArg)) { - if (fSign < 0) { - amd::Os::printf(fmtF.data(), "-nan"); - } else { - amd::Os::printf(fmtF.data(), "nan"); - } - } else if (hlModifier) { - amd::Os::printf(hlFmt.data(), fArg); - } else { - amd::Os::printf(fmt.data(), fArg); - } - } else { - bool hhModifier = (strstr(fmt.c_str(), "hh") != NULL); - if (hhModifier) { - // current implementation of printf in gcc 4.5.2 runtime libraries, doesn`t recognize - // "hh" modifier ==> - // argument should be explicitly converted to unsigned char (uchar) before printing and - // fmt should be updated not to contain "hh" modifier - std::string hhFmt = fmt; - hhFmt.erase(hhFmt.find_first_of("h"), 2); - amd::Os::printf(hhFmt.data(), *(reinterpret_cast(argument))); - } else if (hlModifier) { - amd::Os::printf(hlFmt.data(), *argument); - } else { - amd::Os::printf(fmt.data(), *argument); - } - } - break; - case 8: - if (printFloat) { - if (hlModifier) { - amd::Os::printf(hlFmt.data(), *(reinterpret_cast(argument))); - } else { - amd::Os::printf(fmt.data(), *(reinterpret_cast(argument))); - } - } else { - std::string out = fmt; - // Use 'll' for 64 bit printf - out.insert((out.size() - 1), 1, 'l'); - amd::Os::printf(out.data(), *(reinterpret_cast(argument))); - } - break; - case ConstStr: { - const char* str = reinterpret_cast(argument); - amd::Os::printf(fmt.data(), str); - } break; - default: - amd::Os::printf("Error: Unsupported data size for PrintfDbg. %d bytes", - static_cast(size)); - return 0; - } - } - fflush(stdout); - return copiedBytes; -} - -void PrintfDbg::outputDbgBuffer(const device::PrintfInfo& info, const uint32_t* workitemData, - size_t& i) const { - static const char* specifiers = "cdieEfgGaosuxXp"; - static const char* modifiers = "hl"; - static const char* special = "%n"; - static const std::string sepStr = "%s"; - const uint32_t* s = workitemData; - size_t pos = 0; - - // Find the format string - std::string str = info.fmtString_; - std::string fmt; - size_t posStart, posEnd; - - // Print all arguments - // Note: the following code walks through all arguments, provided by the kernel and - // finds the corresponding specifier in the format string. - // Then it splits the original string into substrings with a single specifier and - // uses standard PrintfDbg() to print each argument - for (uint j = 0; j < info.arguments_.size(); ++j) { - do { - posStart = str.find_first_of("%", pos); - if (posStart != std::string::npos) { - posStart++; - // Erase all spaces after % - while (str[posStart] == ' ') { - str.erase(posStart, 1); - } - size_t tmp = str.find_first_of(special, posStart); - size_t tmp2 = str.find_first_of(specifiers, posStart); - // Special cases. Special symbol is located before any specifier - if (tmp < tmp2) { - posEnd = posStart + 1; - fmt = str.substr(pos, posEnd - pos); - fmt.erase(posStart - pos - 1, 1); - pos = posStart = posEnd; - outputArgument(sepStr, false, ConstStr, reinterpret_cast(fmt.data())); - continue; - } - break; - } else if (pos < str.length()) { - outputArgument(sepStr, false, ConstStr, - reinterpret_cast((str.substr(pos)).data())); - } - } while (posStart != std::string::npos); - - if (posStart != std::string::npos) { - bool printFloat = false; - int vectorSize = 0; - size_t idPos = 0; - - // Search for PrintfDbg specifier in the format string. - // It will be a split point for the output - posEnd = str.find_first_of(specifiers, posStart); - if (posEnd == std::string::npos) { - pos = posStart = posEnd; - break; - } - posEnd++; - - size_t curPos = posEnd; - vectorSize = checkVectorSpecifier(str, posStart, curPos); - - // Get substring from the last position to the current specifier - fmt = str.substr(pos, posEnd - pos); - - // Readjust the string pointer if PrintfDbg outputs a vector - if (vectorSize != 0) { - size_t posVecSpec = fmt.length() - (curPos + 1); - size_t posVecMod = fmt.find_first_of(modifiers, posVecSpec + 1); - size_t posMod = str.find_first_of(modifiers, posStart); - if (posMod < posEnd) { - fmt = fmt.erase(posVecSpec, posVecMod - posVecSpec); - } else { - fmt = fmt.erase(posVecSpec, curPos); - } - idPos = posStart - pos - 1; - } - pos = posStart = posEnd; - - // Find out if the argument is a float - printFloat = checkFloat(fmt); - - // Is it a scalar value? - if (vectorSize == 0) { - size_t length = outputArgument(fmt, printFloat, info.arguments_[j], &s[i]); - if (0 == length) { - return; - } - i += amd::alignUp(length, sizeof(uint32_t)) / sizeof(uint32_t); - } else { - // 3-component vector's size is defined as 4 * size of each scalar component - size_t elemSize = info.arguments_[j] / (vectorSize == 3 ? 4 : vectorSize); - size_t k = i * sizeof(uint32_t); - std::string elementStr = fmt.substr(idPos, fmt.size()); - - // Print first element with full string - if (0 == outputArgument(fmt, printFloat, elemSize, &s[i])) { - return; - } - - // Print other elemnts with separator if available - for (int e = 1; e < vectorSize; ++e) { - const char* t = reinterpret_cast(s); - // Output the vector separator - outputArgument(sepStr, false, ConstStr, reinterpret_cast(Separator)); - - // Output the next element - outputArgument(elementStr, printFloat, elemSize, - reinterpret_cast(&t[k + e * elemSize])); - } - i += (amd::alignUp(info.arguments_[j], sizeof(uint32_t))) / sizeof(uint32_t); - } - } - } - - if (pos != std::string::npos) { - fmt = str.substr(pos, str.size() - pos); - outputArgument(sepStr, false, ConstStr, reinterpret_cast(fmt.data())); - } -} - -bool PrintfDbg::clearWorkitems(VirtualGPU& gpu, size_t idxStart, size_t number) const { - // Go through all locations for every thread and copy 1 - for (uint i = idxStart; i < idxStart + number; ++i) { - amd::Coord3D dst(i * wiDbgSize(), 0, 0); - amd::Coord3D size(sizeof(uint32_t), 0, 0); - - // Copy 1 into the corresponding location in the debug buffer - if (!initCntValue_.partialMemCopyTo(gpu, amd::Coord3D(0, 0, 0), dst, size, *dbgBuffer_)) { - return false; - } - } - return true; -} - -uint32_t* PrintfDbg::mapWorkitem(VirtualGPU& gpu, size_t idx, bool* realloc) { - uint32_t wiSize = 0; - amd::Coord3D src(idx * wiDbgSize(), 0, 0); - xferBufRead_ = &(dev().xferRead().acquire()); - - // Copy workitem size from the corresponding location in the debug buffer - if (!dbgBuffer_->partialMemCopyTo(gpu, src, amd::Coord3D(0, 0, 0), - amd::Coord3D(sizeof(uint32_t), 0, 0), *xferBufRead_)) { - return NULL; - } - - // Get memory pointer to the satged buffer - uint32_t* workitem = reinterpret_cast(xferBufRead_->map(&gpu)); - if (NULL == workitem) { - return NULL; - } - - // Copy size value - wiSize = *workitem; - xferBufRead_->unmap(&gpu); - - // Check if the cuurent workitem almost reached the size limit - if ((wiDbgSize() - static_cast(wiSize)) < 3) { - *realloc = true; - } - - // If the current workitem had any output then get the data - if ((wiSize > 1) && (wiSize <= wiDbgSize())) { - amd::Coord3D size(wiSize * sizeof(uint32_t), 0, 0); - - // Copy the current workitem output data to the staged buffer - if (!dbgBuffer_->partialMemCopyTo(gpu, src, amd::Coord3D(0, 0, 0), size, *xferBufRead_) || - // Clear the write pointer back to index 1 for the current workitem - !clearWorkitems(gpu, idx, 1)) { - LogError("Reading the workitem data failed!"); - return NULL; - } - - // Get a pointer to the workitem data - uint32_t* workitem = reinterpret_cast(xferBufRead_->map(&gpu)); - - return workitem; - } - - return NULL; -} - -void PrintfDbg::unmapWorkitem(VirtualGPU& gpu, const uint32_t* workitemData) const { - if (NULL != workitemData) { - xferBufRead_->unmap(&gpu); - } - - dev().xferRead().release(gpu, *xferBufRead_); -} - -bool PrintfDbgHSA::init(VirtualGPU& gpu, bool printfEnabled) { - // Set up debug output buffer (if printf active) - if (printfEnabled) { - if (!allocate()) { - return false; - } - - // The first two DWORDs in the printf buffer are as follows: - // First DWORD = Offset to where next information is to - // be written, initialized to 0 - // Second DWORD = Number of bytes available for printf data - // = buffer size – 2*sizeof(uint32_t) - const uint8_t initSize = 2 * sizeof(uint32_t); - uint8_t sysMem[initSize]; - memset(sysMem, 0, initSize); - uint32_t dbgBufferSize = dbgBuffer_->size() - initSize; - memcpy(&sysMem[4], &dbgBufferSize, sizeof(dbgBufferSize)); - - // Copy offset and number of bytes available for printf data - // into the corresponding location in the debug buffer - dbgBuffer_->writeRawData(gpu, initSize, sysMem, true); - } - return true; -} - -bool PrintfDbgHSA::output(VirtualGPU& gpu, bool printfEnabled, - const std::vector& printfInfo) { - if (printfEnabled) { - uint32_t offsetSize = 0; - xferBufRead_ = &(dev().xferRead().acquire()); - - // Copy offset from the first DWORD in the debug buffer - if (!dbgBuffer_->partialMemCopyTo(gpu, amd::Coord3D(0, 0, 0), amd::Coord3D(0, 0, 0), - amd::Coord3D(sizeof(uint32_t), 0, 0), *xferBufRead_)) { - return false; - } - - // Get memory pointer to the satged buffer - uint32_t* dbgBufferPtr = reinterpret_cast(xferBufRead_->map(&gpu)); - if (NULL == dbgBufferPtr) { - return false; - } - - offsetSize = *dbgBufferPtr; - xferBufRead_->unmap(&gpu); - - if (offsetSize == 0) { - LogInfo("The printf buffer is empty!"); - dev().xferRead().release(gpu, *xferBufRead_); - return true; - } - - size_t bufSize = dev().xferRead().bufSize(); - size_t copySize = offsetSize; - while (copySize != 0) { - // Copy the buffer data (i.e., the printfID followed by the - // argument data for each printf call in th kernel) to the staged buffer - if (!dbgBuffer_->partialMemCopyTo( - gpu, amd::Coord3D(2 * sizeof(uint32_t) + offsetSize - copySize, 0, 0), - amd::Coord3D(0, 0, 0), std::min(copySize, bufSize), *xferBufRead_)) { - return false; - } - - // Get a pointer to the buffer data - dbgBufferPtr = reinterpret_cast(xferBufRead_->map(&gpu)); - if (NULL == dbgBufferPtr) { - return false; - } - - uint sb = 0; - uint sbt = 0; - - // parse the debug buffer - while (sbt < copySize) { - if (*dbgBufferPtr >= printfInfo.size()) { - LogError("Couldn't find the reported PrintfID!"); - return false; - } - const device::PrintfInfo& info = printfInfo[(*dbgBufferPtr)]; - sb += sizeof(uint32_t); - for (const auto& it : info.arguments_) { - sb += it; - } - - if (sbt + sb > bufSize) { - break; // Need new portion of data in staging buffer - } - - size_t idx = 1; - // There's something in the debug buffer - outputDbgBuffer(info, dbgBufferPtr, idx); - - sbt += sb; - dbgBufferPtr += sb / sizeof(uint32_t); - sb = 0; - } - - copySize -= sbt; - xferBufRead_->unmap(&gpu); - } - - dev().xferRead().release(gpu, *xferBufRead_); - } - - return true; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuprintf.hpp b/projects/clr/rocclr/device/gpu/gpuprintf.hpp deleted file mode 100644 index 2ef52079f2..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuprintf.hpp +++ /dev/null @@ -1,183 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUPRINTFDBG_HPP_ -#define GPUPRINTFDBG_HPP_ - -#include "device/gpu/gpumemory.hpp" - -/*! \addtogroup GPU GPU Device Implementation - * @{ - */ -#ifndef isinf -#ifdef _MSC_VER -#define isinf(X) (!_finite(X) && !_isnan(X)) -#endif //_MSC_VER -#endif // isinf - -#ifndef isnan -#ifdef _MSC_VER -#define isnan(X) (_isnan(X)) -#endif //_MSC_VER -#endif // isnan - -#ifndef copysign -#ifdef _MSC_VER -#define copysign(X, Y) (_copysign(X, Y)) -#endif //_MSC_VER -#endif // copysign - -//! GPU Device Implementation -namespace gpu { - -class Kernel; -class VirtualGPU; -class Memory; - -class PrintfDbg : public amd::HeapObject { - public: - //! Debug buffer size per workitem - static constexpr uint WorkitemDebugSize = 4096; - - //! Default constructor - PrintfDbg(Device& device, FILE* file = NULL); - - //! Destructor - ~PrintfDbg(); - - //! Creates the PrintfDbg object - bool create(); - - //! Initializes the debug buffer before kernel's execution - bool init(VirtualGPU& gpu, //!< Virtual GPU object - bool printfEnabled, //!< checks for printf - const amd::NDRange& size //!< Kernel's workload - ); - - //! Prints the kernel's debug informaiton from the buffer - bool output(VirtualGPU& gpu, //!< Virtual GPU object - bool printfEnabled, //!< checks for printf - const amd::NDRange& size, //!< Kernel's workload - const std::vector& printfInfo //!< printf info - ); - - //! Returns the debug buffer offset - uint64_t bufOffset() const; - - //! Debug buffer size per workitem - size_t wiDbgSize() const { return wiDbgSize_; } - - //! Returns debug buffer object - Memory* dbgBuffer() const { return dbgBuffer_; } - - protected: - Memory* dbgBuffer_; //!< Buffer to hold debug output - FILE* dbgFile_; //!< Debug file - Device& gpuDevice_; //!< GPU device object - Memory* xferBufRead_; //!< Transfer buffer for the dump read - - //! Gets GPU device object - Device& dev() const { return gpuDevice_; } - - //! Allocates the debug buffer - bool allocate(bool realloc = false //!< If TRUE then reallocate the debug memory - ); - - //! Returns TRUE if a float value has to be printed - bool checkFloat(const std::string& fmt //!< Format string - ) const; - - //! Returns TRUE if a string value has to be printed - bool checkString(const std::string& fmt //!< Format string - ) const; - - //! Finds the specifier in the format string - int checkVectorSpecifier(const std::string& fmt, //!< Format string - size_t startPos, //!< Start position for processing - size_t& curPos //!< End position for processing - ) const; - - //! Outputs an argument - size_t outputArgument(const std::string& fmt, //!< Format strint - bool printFloat, //!< Argument is a float value - size_t size, //!< Argument's size - const uint32_t* argument //!< Argument's location - ) const; - - //! Displays the PrintfDbg - void outputDbgBuffer(const device::PrintfInfo& info,//!< printf info - const uint32_t* workitemData, //!< The PrintfDbg dump buffer - size_t& i //!< index to the data in the buffer - ) const; - - private: - //! Disable copy constructor - PrintfDbg(const PrintfDbg&); - - //! Disable assignment - PrintfDbg& operator=(const PrintfDbg&); - - //! Returns the pointer to the workitem data block - bool clearWorkitems(VirtualGPU& gpu, //!< Virtual GPU object - size_t idxStart, //!< Workitem global index start - size_t number //!< Number of workitems to clear - ) const; - - //! Returns the pointer to the workitem data block - uint32_t* mapWorkitem(VirtualGPU& gpu, //!< Virtual GPU object - size_t idx, //!< Workitem global index - bool* realloc //!< Returns TRUE if workitem reached the buffer limit - ); - - //! Unamp the staged buffer - void unmapWorkitem(VirtualGPU& gpu, //!< Virtual GPU object - const uint32_t* workitemData //!< The PrintfDbg dump buffer - ) const; - - size_t wiDbgSize_; //!< Workitem debug size - Memory initCntValue_; //!< Initialized count value -}; -class PrintfDbgHSA : public PrintfDbg { - public: - //! Default constructor - PrintfDbgHSA(Device& device, FILE* file = NULL) : PrintfDbg(device, file) {} - - //! Initializes the debug buffer before kernel's execution - bool init(VirtualGPU& gpu, //!< Virtual GPU object - bool printfEnabled //!< checks for printf - ); - - //! Prints the kernel's debug informaiton from the buffer - bool output(VirtualGPU& gpu, //!< Virtual GPU object - bool printfEnabled, //!< checks for printf - const std::vector& printfInfo //!< printf info - ); - - private: - //! Disable copy constructor - PrintfDbgHSA(const PrintfDbgHSA&); - - //! Disable assignment - PrintfDbgHSA& operator=(const PrintfDbgHSA&); -}; - -/*@}*/} // namespace gpu - -#endif /*GPUPRINTFDBG_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpuprogram.cpp b/projects/clr/rocclr/device/gpu/gpuprogram.cpp deleted file mode 100644 index 60830ffcc7..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuprogram.cpp +++ /dev/null @@ -1,1989 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "os/os.hpp" -#include "utils/flags.hpp" -#include "aclTypes.h" -#include "utils/amdilUtils.hpp" -#include "utils/bif_section_labels.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpublit.hpp" -#include "macrodata.h" -#include "MDParser/AMDILMDInterface.h" -#include "utils/options.hpp" -#include "hsa.h" -#include "hsa_ext_image.h" -#include "amd_hsa_loader.hpp" - -#include -#include -#include -#include -#include -#include - -namespace gpu { - -const aclTargetInfo& NullProgram::info() { - acl_error err; - info_ = amd::Hsail::GetTargetInfo(gpuNullDevice().settings().use64BitPtr_ ? "amdil64" : "amdil", - nullptr, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclGetTargetInfo failed"); - } - return info_; -} - -NullProgram::~NullProgram() { - // Destroy all ILFunc objects - freeAllILFuncs(); - releaseClBinary(); -} - -bool NullProgram::isCalled(const ILFunc* base, const ILFunc* func) { - // Loop through all functions, which will be called from the base one - for (size_t i = 0; i < base->calls_.size(); ++i) { - assert(base->calls_[i] != base && "recursion"); - // Check if the current function is the one - if (base->calls_[i] == func) { - return true; - } - // We have to use a recursive method to make sure it's not called inside - else if (isCalled(base->calls_[i], func)) { - return true; - } - } - return false; -} - -uint ILFunc::totalHwPrivateUsage() { - if (totalHwPrivateSize_ >= 0) return totalHwPrivateSize_; - - uint maxChildUsage = 0; - for (size_t i = 0; i < calls_.size(); ++i) { - uint childUsage = calls_[i]->totalHwPrivateUsage(); - if (childUsage > maxChildUsage) maxChildUsage = childUsage; - } - totalHwPrivateSize_ = hwPrivateSize_ + maxChildUsage; - return totalHwPrivateSize_; -} - -void NullProgram::patchMain(std::string& kernel, uint index) { - std::string callPatch = "call "; - char sym; - - // Create the patch string - while (index) { - sym = (index % 10) + 0x30; - callPatch.insert(5, &sym, 1); - index /= 10; - } - callPatch += ";"; - - // Patch the program - kernel.replace(patch_, callPatch.size(), callPatch); -} - -NullKernel* Program::createKernel(const std::string& name, const Kernel::InitData* initData, - const std::string& code, const std::string& metadata, - bool* created, const void* binaryCode, size_t binarySize) { - amd::option::Options* options = getCompilerOptions(); - uint64_t start_time = 0; - if (options->oVariables->EnableBuildTiming) { - start_time = amd::Os::timeNanos(); - } - - *created = false; - // Create a GPU kernel - Kernel* gpuKernel = new Kernel(name, gpuDevice(), *this, initData); - - if (gpuKernel == NULL) { - buildLog_ += "new Kernel() failed"; - LogPrintfError("new Kernel() failed for kernel %s!", name.c_str()); - return NULL; - } else if (gpuKernel->create(code, metadata, binaryCode, binarySize)) { - // Add kernel to the program - kernels()[gpuKernel->name()] = gpuKernel; - buildLog_ += gpuKernel->buildLog(); - } else { - buildError_ = gpuKernel->buildError(); - buildLog_ += gpuKernel->buildLog(); - delete gpuKernel; - LogPrintfError("Kernel creation failed for kernel %s!", name.c_str()); - return NULL; - } - - if (options->oVariables->EnableBuildTiming) { - std::stringstream tmp_ss; - tmp_ss << " Time for creating kernel (" << name - << ") : " << (amd::Os::timeNanos() - start_time) / 1000ULL << " us\n"; - buildLog_ += tmp_ss.str(); - } - - *created = true; - return static_cast(gpuKernel); -} - -bool NullProgram::linkImpl(amd::option::Options* options) { - if (llvmBinary_.empty()) { - // We are using either CL binary or IL directly. - bool hasRecompiled; - if (ilProgram_.empty()) { - // Setup elfIn() and try to load ISA from binary - // This elfIn() will be released at the end of build by finiBuild(). - if (!clBinary()->setElfIn()) { - buildLog_ += "Internal error: Setting input OCL binary failed!\n"; - LogError("Setting input OCL binary failed"); - return false; - } - bool loadSuccess = false; - if (!options->oVariables->ForceLLVM) { - loadSuccess = loadBinary(&hasRecompiled); - } - if (!loadSuccess && (options->oVariables->UseDebugIL && !options->oVariables->ForceLLVM)) { - buildLog_ += "Internal error: Loading OpenCL binary under -use-debugil failed!\n"; - LogError("Loading OCL binary failed under -use-debugil"); - return false; - } - if (loadSuccess) { - if (hasRecompiled) { - char* section; - size_t sz; - if (clBinary()->saveSOURCE() && - clBinary()->elfIn()->getSection(amd::Elf::SOURCE, §ion, &sz)) { - clBinary()->elfOut()->addSection(amd::Elf::SOURCE, section, sz); - } - if (clBinary()->saveLLVMIR()) { - if (clBinary()->loadLlvmBinary(llvmBinary_, elfSectionType_) && - (!llvmBinary_.empty())) { - clBinary()->elfOut()->addSection(elfSectionType_, llvmBinary_.data(), - llvmBinary_.size()); - } - } - - setType(TYPE_EXECUTABLE); - if (!clBinary()->createElfBinary(options->oVariables->BinEncrypt, type())) { - buildLog_ += "Internal error: Failed to create OpenCL binary!\n"; - LogError("Failed to create OpenCL binary"); - return false; - } - } else { - // The original binary is good and reuse it. - // Release the new binary if there is. - clBinary()->restoreOrigBinary(); - } - return true; - } else if (clBinary()->loadLlvmBinary(llvmBinary_, elfSectionType_) && - clBinary()->isRecompilable(llvmBinary_, amd::Elf::CAL_PLATFORM)) { - char* section; - size_t sz; - - // Clean up and remove all the content generated before - if (!clBinary()->clearElfOut()) { - buildLog_ += "Internal error: Resetting OpenCL Binary failed!\n"; - LogError("Resetting output OCL binary failed"); - return false; - } - - if (clBinary()->saveSOURCE() && - clBinary()->elfIn()->getSection(amd::Elf::SOURCE, §ion, &sz)) { - clBinary()->elfOut()->addSection(amd::Elf::SOURCE, section, sz); - } - if (clBinary()->saveLLVMIR()) { - clBinary()->elfOut()->addSection(elfSectionType_, llvmBinary_.data(), llvmBinary_.size()); - } - } else { - buildLog_ += "Internal error: Input OpenCL binary is not for the target!\n"; - LogError("OCL Binary isn't good for the target"); - return false; - } - } - } - - if (!llvmBinary_.empty()) { - // Compile llvm binary to the IL source code - // This is link/OPT/Codegen part of compiler. - int32_t iErr = compileBinaryToIL(options); - if (iErr != CL_SUCCESS) { - buildLog_ += "Error: Compilation from LLVMIR binary to IL text failed!"; - LogError(buildLog_.c_str()); - return false; - } - } - - if (!ilProgram_.empty() && options->oVariables->EnableDebug) { - // Lets parse out the dwarf debug information and store it in the elf - llvm::CompUnit compilation(ilProgram_); - std::string debugILStr = compilation.getILStr(); - const char* dbgSec = debugILStr.c_str(); - size_t dbgSize = debugILStr.size(); - // Add an IL section that contains debug information and is the - // output of LLVM codegen. - clBinary()->elfOut()->addSection(amd::Elf::ILDEBUG, dbgSec, dbgSize); - - if ((dbgSize > 0) && options->isDumpFlagSet(amd::option::DUMP_DEBUGIL)) { - std::string debugilWithLine; - size_t b = 1; - int linenum = 0; - char cstr[9]; - cstr[8] = 0; - while (b != std::string::npos) { - size_t e = debugILStr.find_first_of("\n", b); - if (e != std::string::npos) { - ++e; - } - sprintf(&cstr[0], "%5x: ", linenum); - debugilWithLine.append(cstr); - debugilWithLine.append(debugILStr.substr(b, e - b)); - b = e; - ++linenum; - } - std::string debugilFileName = options->getDumpFileName(".debugil"); - std::fstream f; - f.open(debugilFileName.c_str(), (std::fstream::out | std::fstream::binary)); - f.write(debugilWithLine.c_str(), debugilWithLine.size()); - f.close(); - } - - for (unsigned x = 0; x < llvm::AMDILDwarf::DEBUG_LAST; ++x) { - dbgSec = compilation.getDebugData()->getDwarfBitstream( - static_cast(x), dbgSize); - // Do not create an elf section if the size of the section is - // 0. - if (!dbgSize) { - continue; - } - clBinary()->elfOut()->addSection( - static_cast(x + amd::Elf::DEBUG_INFO), dbgSec, dbgSize); - } - - } - - // Create kernel objects - if (!ilProgram_.empty() && parseKernels(ilProgram_)) { - // Loop through all possible kernels - for (size_t i = 0; i < funcs_.size(); ++i) { - ILFunc* baseFunc = funcs_[i]; - // Make sure we have a Kernel function, but not Intrinsic or Simple - if (baseFunc->state_ == ILFunc::Kernel) { - size_t metadataSize = baseFunc->metadata_.end_ - baseFunc->metadata_.begin_; - std::string kernel = ilProgram_; - std::string metadataStr; - std::vector notCalled; - std::vector called; - std::unordered_map macros; - size_t j; - Kernel::InitData initData = {0}; - - // Fill the list of not used functions, relativly to the current - for (j = 0; j < funcs_.size(); ++j) { - if ((i != j) && - ((funcs_[j]->state_ == ILFunc::Regular) || (funcs_[j]->state_ == ILFunc::Kernel))) { - if (!isCalled(baseFunc, funcs_[j])) { - notCalled.push_back(funcs_[j]); - } else { - called.push_back(funcs_[j]); - } - } - } - - // Get the metadata string for the current kernel - metadataStr.insert(0, kernel, baseFunc->metadata_.begin_, metadataSize); - - std::vector rangeList; - // Remove unused kernels, starting from the end - for (j = notCalled.size(); j > 0; --j) { - ILFunc* func = notCalled[j - 1]; - std::vector::iterator it; - for (it = rangeList.begin(); it != rangeList.end(); ++it) { - if ((*it)->begin_ < func->metadata_.begin_) { - assert((*it)->begin_ < func->code_.begin_ && - "code and metadata not next to each other"); - break; - } - assert((*it)->begin_ >= func->code_.begin_ && - "code and metadata not next to each other"); - } - assert(func->metadata_.begin_ > func->code_.begin_ && "code after metadata"); - if (it == rangeList.end()) { - rangeList.push_back(&func->metadata_); - rangeList.push_back(&func->code_); - } else { - it = rangeList.insert(it, &func->code_); - rangeList.insert(it, &func->metadata_); - } - } - for (j = 0; j < rangeList.size(); ++j) { - const ILFunc::SourceRange* range = rangeList[j]; - kernel.erase(range->begin_, range->end_ - range->begin_); - } - - // Patch the main program with a call to the current kernel - patchMain(kernel, baseFunc->index_); - - // Add macros at the top, loop through all available functions - // for this kernel - for (j = 0; j <= called.size(); ++j) { - ILFunc* func = (j < called.size()) ? called[j] : baseFunc; - for (size_t l = func->macros_.size(); l > 0; --l) { - int lines; - int idx = static_cast(func->macros_[l - 1]); - const char** macro = amd::MacroDBGetMacro(&lines, idx); - - // Make sure we didn't place this macro already - if (macros[idx] == NULL) { - macros[idx] = macro; - // Do we have a valid macro? - if ((lines == 0) || (macro == NULL)) { - buildLog_ += "Error: undefined macro!\n"; - LogPrintfError("Metadata reports undefined macro %d!", idx); - return false; - } else { - // Add the macro to the IL source - for (int k = 0; k < lines; ++k) { - kernel.insert(0, macro[k], strlen(macro[k])); - } - } - } - } - // Accumulate all emulated local and private sizes, - // necessary for the kernel execution - initData.localSize_ += func->localSize_; - - // Accumulate all HW local and private sizes, - // necessary for the kernel execution - initData.hwLocalSize_ += func->hwLocalSize_; - initData.hwPrivateSize_ += func->hwPrivateSize_; - initData.flags_ |= func->flags_; - } - initData.privateSize_ = baseFunc->totalHwPrivateUsage(); - amdilUtils::changePrivateUAVLength(kernel, initData.privateSize_); - - // Create a GPU kernel - bool created; - NullKernel* gpuKernel = - createKernel(baseFunc->name_, &initData, kernel.data(), metadataStr, &created); - if (!created) { - buildLog_ += "Error: Creating kernel " + baseFunc->name_ + " failed!\n"; - LogError(buildLog_.c_str()); - return false; - } - - // Add the current kernel to the binary - if (!clBinary()->storeKernel(baseFunc->name_, gpuKernel, &initData, metadataStr, kernel)) { - buildLog_ += "Internal error: adding a kernel into OpenCL binary failed!\n"; - return false; - } - } - } - - setType(TYPE_EXECUTABLE); - if (!createBinary(options)) { - buildLog_ += "Intenral error: creating OpenCL binary failed\n"; - return false; - } - - // Destroy all ILFunc objects - freeAllILFuncs(); - ilProgram_.clear(); - return true; - } - return false; -} - -bool NullProgram::linkImpl(const std::vector& inputPrograms, - amd::option::Options* options, bool createLibrary) { - std::vector llvmBinaries(inputPrograms.size()); - std::vector elfSectionType(inputPrograms.size()); - auto it = inputPrograms.cbegin(); - const auto itEnd = inputPrograms.cend(); - for (size_t i = 0; it != itEnd; ++it, ++i) { - NullProgram* program = (NullProgram*)*it; - - if (program->llvmBinary_.empty()) { - if (program->clBinary() == NULL) { - buildLog_ += "Internal error: Input program not compiled!\n"; - LogError("Loading compiled input object failed"); - return false; - } - - // We are using CL binary directly. - // Setup elfIn() and try to load llvmIR from binary - // This elfIn() will be released at the end of build by finiBuild(). - if (!program->clBinary()->setElfIn()) { - buildLog_ += "Internal error: Setting input OCL binary failed!\n"; - LogError("Setting input OCL binary failed"); - return false; - } - if (!program->clBinary()->loadLlvmBinary(program->llvmBinary_, program->elfSectionType_)) { - buildLog_ += "Internal error: Failed loading compiled binary!\n"; - LogError("Bad OCL Binary"); - return false; - } - - if (!program->clBinary()->isRecompilable(program->llvmBinary_, amd::Elf::CAL_PLATFORM)) { - buildLog_ += - "Internal error: Input OpenCL binary is not" - " for the target!\n"; - LogError("OCL Binary isn't good for the target"); - return false; - } -#if 0 - // TODO: copy .source over to output program - char *section; - size_t sz; - - if (clBinary()->saveSOURCE() && - clBinary()->elfIn()->getSection(amd::Elf::SOURCE, §ion, &sz)) { - clBinary()->elfOut()->addSection(amd::Elf::SOURCE, section, sz); - } -#endif - } - - llvmBinaries[i] = &program->llvmBinary_; - elfSectionType[i] = program->elfSectionType_; - } - - acl_error err; - aclTargetInfo aclinfo = info(); - aclBinaryOptions binOpts = {0}; - binOpts.struct_size = sizeof(binOpts); - binOpts.elfclass = aclinfo.arch_id == aclAMDIL64 ? ELFCLASS64 : ELFCLASS32; - binOpts.bitness = ELFDATA2LSB; - binOpts.alloc = &::malloc; - binOpts.dealloc = &::free; - - std::vector libs(llvmBinaries.size(), NULL); - for (size_t i = 0; i < libs.size(); ++i) { - libs[i] = amd::Hsail::BinaryInit(sizeof(aclBinary), &aclinfo, &binOpts, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclBinaryInit failed"); - break; - } - - _bif_sections_enum_0_8 aclTypeUsed; - if (elfSectionType[i] == amd::Elf::SPIRV) { - aclTypeUsed = aclSPIRV; - } else if (elfSectionType[i] == amd::Elf::SPIR) { - aclTypeUsed = aclSPIR; - } else { - aclTypeUsed = aclLLVMIR; - } - err = amd::Hsail::InsertSection(gpuNullDevice().amdilCompiler(), libs[i], llvmBinaries[i]->data(), - llvmBinaries[i]->size(), aclTypeUsed); - if (err != ACL_SUCCESS) { - LogWarning("aclInsertSection failed"); - break; - } - - // temporary solution to synchronize buildNo between runtime and complib - // until we move runtime inside complib - ((amd::option::Options*)libs[i]->options)->setBuildNo(options->getBuildNo()); - } - - - if (libs.size() > 0 && err == ACL_SUCCESS) do { - unsigned int numLibs = libs.size() - 1; - - if (numLibs > 0) { - err = amd::Hsail::Link(gpuNullDevice().amdilCompiler(), libs[0], numLibs, &libs[1], ACL_TYPE_LLVMIR_BINARY, - "-create-library", NULL); - - buildLog_ += amd::Hsail::GetCompilerLog(gpuNullDevice().amdilCompiler()); - - if (err != ACL_SUCCESS) { - LogWarning("aclLink failed"); - break; - } - } - - size_t size = 0; - _bif_sections_enum_0_8 aclTypeUsed; - if (elfSectionType[0] == amd::Elf::SPIRV && numLibs == 0) { - aclTypeUsed = aclSPIRV; - } else if (elfSectionType[0] == amd::Elf::SPIR && numLibs == 0) { - aclTypeUsed = aclSPIR; - } else { - aclTypeUsed = aclLLVMIR; - } - const void* llvmir = amd::Hsail::ExtractSection(gpuNullDevice().amdilCompiler(), libs[0], &size, aclTypeUsed, &err); - if (err != ACL_SUCCESS) { - LogWarning("aclExtractSection failed"); - break; - } - - llvmBinary_.assign(reinterpret_cast(llvmir), size); - elfSectionType_ = amd::Elf::LLVMIR; - } while (0); - - std::for_each(libs.begin(), libs.end(), std::ptr_fun(amd::Hsail::BinaryFini)); - - if (err != ACL_SUCCESS) { - buildLog_ += "Error: linking llvm modules failed!"; - return false; - } - - if (clBinary()->saveLLVMIR()) { - clBinary()->elfOut()->addSection(amd::Elf::LLVMIR, llvmBinary_.data(), llvmBinary_.size()); - // store the original link options - clBinary()->storeLinkOptions(linkOptions_); - - clBinary()->storeCompileOptions(compileOptions_); - } - - // skip the rest if we are building an opencl library - if (createLibrary) { - setType(TYPE_LIBRARY); - if (!createBinary(options)) { - buildLog_ += "Intenral error: creating OpenCL binary failed\n"; - return false; - } - - return true; - } - - // Compile llvm binary to the IL source code - // This is link/OPT/Codegen part of compiler. - int32_t iErr = compileBinaryToIL(options); - if (iErr != CL_SUCCESS) { - buildLog_ += "Error: Compilation from LLVMIR binary to IL text failed!"; - LogError(buildLog_.c_str()); - return false; - } - - if (!ilProgram_.empty() && options->oVariables->EnableDebug) { - // Lets parse out the dwarf debug information and store it in the elf - llvm::CompUnit compilation(ilProgram_); - std::string debugILStr = compilation.getILStr(); - const char* dbgSec = debugILStr.c_str(); - size_t dbgSize = debugILStr.size(); - // Add an IL section that contains debug information and is the - // output of LLVM codegen. - clBinary()->elfOut()->addSection(amd::Elf::ILDEBUG, dbgSec, dbgSize); - - if ((dbgSize > 0) && options->isDumpFlagSet(amd::option::DUMP_DEBUGIL)) { - std::string debugilWithLine; - size_t b = 1; - int linenum = 0; - char cstr[9]; - cstr[8] = 0; - while (b != std::string::npos) { - size_t e = debugILStr.find_first_of("\n", b); - if (e != std::string::npos) { - ++e; - } - sprintf(&cstr[0], "%5x: ", linenum); - debugilWithLine.append(cstr); - debugilWithLine.append(debugILStr.substr(b, e - b)); - b = e; - ++linenum; - } - std::string debugilFileName = options->getDumpFileName(".debugil"); - std::fstream f; - f.open(debugilFileName.c_str(), (std::fstream::out | std::fstream::binary)); - f.write(debugilWithLine.c_str(), debugilWithLine.size()); - f.close(); - } - - for (unsigned x = 0; x < llvm::AMDILDwarf::DEBUG_LAST; ++x) { - dbgSec = compilation.getDebugData()->getDwarfBitstream( - static_cast(x), dbgSize); - // Do not create an elf section if the size of the section is - // 0. - if (!dbgSize) { - continue; - } - clBinary()->elfOut()->addSection( - static_cast(x + amd::Elf::DEBUG_INFO), dbgSec, dbgSize); - } - - } - - // Create kernel objects - if (!ilProgram_.empty() && parseKernels(ilProgram_)) { - // Loop through all possible kernels - for (size_t i = 0; i < funcs_.size(); ++i) { - ILFunc* baseFunc = funcs_[i]; - // Make sure we have a Kernel function, but not Intrinsic or Simple - if (baseFunc->state_ == ILFunc::Kernel) { - size_t metadataSize = baseFunc->metadata_.end_ - baseFunc->metadata_.begin_; - std::string kernel = ilProgram_; - std::string metadataStr; - std::vector notCalled; - std::vector called; - std::unordered_map macros; - size_t j; - Kernel::InitData initData = {0}; - - // Fill the list of not used functions, relativly to the current - for (j = 0; j < funcs_.size(); ++j) { - if ((i != j) && - ((funcs_[j]->state_ == ILFunc::Regular) || (funcs_[j]->state_ == ILFunc::Kernel))) { - if (!isCalled(baseFunc, funcs_[j])) { - notCalled.push_back(funcs_[j]); - } else { - called.push_back(funcs_[j]); - } - } - } - - // Get the metadata string for the current kernel - metadataStr.insert(0, kernel, baseFunc->metadata_.begin_, metadataSize); - - std::vector rangeList; - // Remove unused kernels, starting from the end - for (j = notCalled.size(); j > 0; --j) { - ILFunc* func = notCalled[j - 1]; - std::vector::iterator it; - for (it = rangeList.begin(); it != rangeList.end(); ++it) { - if ((*it)->begin_ < func->metadata_.begin_) { - assert((*it)->begin_ < func->code_.begin_ && - "code and metadata not next to each other"); - break; - } - assert((*it)->begin_ >= func->code_.begin_ && - "code and metadata not next to each other"); - } - assert(func->metadata_.begin_ > func->code_.begin_ && "code after metadata"); - if (it == rangeList.end()) { - rangeList.push_back(&func->metadata_); - rangeList.push_back(&func->code_); - } else { - it = rangeList.insert(it, &func->code_); - rangeList.insert(it, &func->metadata_); - } - } - for (j = 0; j < rangeList.size(); ++j) { - const ILFunc::SourceRange* range = rangeList[j]; - kernel.erase(range->begin_, range->end_ - range->begin_); - } - - // Patch the main program with a call to the current kernel - patchMain(kernel, baseFunc->index_); - - // Add macros at the top, loop through all available functions - // for this kernel - for (j = 0; j <= called.size(); ++j) { - ILFunc* func = (j < called.size()) ? called[j] : baseFunc; - for (size_t l = func->macros_.size(); l > 0; --l) { - int lines; - int idx = static_cast(func->macros_[l - 1]); - const char** macro = amd::MacroDBGetMacro(&lines, idx); - - // Make sure we didn't place this macro already - if (macros[idx] == NULL) { - macros[idx] = macro; - // Do we have a valid macro? - if ((lines == 0) || (macro == NULL)) { - buildLog_ += "Error: undefined macro!\n"; - LogPrintfError("Metadata reports undefined macro %d!", idx); - return false; - } else { - // Add the macro to the IL source - for (int k = 0; k < lines; ++k) { - kernel.insert(0, macro[k], strlen(macro[k])); - } - } - } - } - // Accumulate all emulated local and private sizes, - // necessary for the kernel execution - initData.localSize_ += func->localSize_; - - // Accumulate all HW local and private sizes, - // necessary for the kernel execution - initData.hwLocalSize_ += func->hwLocalSize_; - initData.hwPrivateSize_ += func->hwPrivateSize_; - initData.flags_ |= func->flags_; - } - initData.privateSize_ = baseFunc->totalHwPrivateUsage(); - amdilUtils::changePrivateUAVLength(kernel, initData.privateSize_); - - // Create a GPU kernel - bool created; - NullKernel* gpuKernel = - createKernel(baseFunc->name_, &initData, kernel.data(), metadataStr, &created); - if (!created) { - buildLog_ += "Error: Creating kernel " + baseFunc->name_ + " failed!\n"; - LogError(buildLog_.c_str()); - return false; - } - - // Add the current kernel to the binary - if (!clBinary()->storeKernel(baseFunc->name_, gpuKernel, &initData, metadataStr, kernel)) { - buildLog_ += "Internal error: adding a kernel into OpenCL binary failed!\n"; - return false; - } - } - } - - setType(TYPE_EXECUTABLE); - if (!createBinary(options)) { - buildLog_ += "Intenral error: creating OpenCL binary failed\n"; - return false; - } - - // Destroy all ILFunc objects - freeAllILFuncs(); - ilProgram_.clear(); - return true; - } - return false; -} - -bool NullProgram::initClBinary() { - if (clBinary_ == NULL) { - clBinary_ = new ClBinary(static_cast(device())); - if (clBinary_ == NULL) { - return false; - } - } - return true; -} - -bool NullProgram::loadBinary(bool* hasRecompiled) { - if (!clBinary()->loadKernels(*this, hasRecompiled)) { - clear(); - return false; - } - return true; -} - -bool NullProgram::initGlobalData(const std::string& source, size_t start) { - size_t pos, dataStart; - - // Find the global data store - dataStart = source.find(";#DATASTART", start); - if (dataStart != std::string::npos) { - uint index = 0; - pos = dataStart + 2; - while (expect(source, &pos, "DATASTART:")) { - uint dataSize = 0; - uint offset; - uint numElements; - size_t posStart; - bool failed = false; - - // Kernel has the global constants - if (!getuint(source, &pos, &index)) { - return false; - } - pos--; - if (expect(source, &pos, ":")) { - // Read the size - if (!getuint(source, &pos, &dataSize)) { - return false; - } - } else { - // Emulated global data store - pos++; - dataSize = index; - index = 0; - } - - if (dataSize == 0) { - return false; - } - - posStart = pos = source.find_first_not_of(";# \n\r", pos); - - char* globalData = new char[dataSize]; - if (globalData == NULL) { - return false; - } - - // Find the global data size - while (!expect(source, &pos, "DATAEND")) { - for (uint i = 0; i < DataTypeTotal; ++i) { - if (expect(source, &pos, DataType[i].tagName_)) { - // Read the offset - if (!getuint(source, &pos, &offset)) { - return false; - } - if (!getuint(source, &pos, &numElements)) { - return false; - } - for (uint j = 0; j < numElements; ++j) { - switch (DataType[i].type_) { - case KernelArg::Float: { - uint32_t* tmp = reinterpret_cast(globalData + offset); - if (!getuintHex(source, &pos, &tmp[j])) { - failed = true; - } - } break; - case KernelArg::Double: { - uint64_t* tmp = reinterpret_cast(globalData + offset); - if (!getuint64Hex(source, &pos, &tmp[j])) { - failed = true; - } - } break; - case KernelArg::Struct: - case KernelArg::Union: - // Struct and Union should be presented as bytes - // Fall through... - case KernelArg::Char: { - uint8_t* tmp = reinterpret_cast(globalData + offset); - uint value; - if (!getuintHex(source, &pos, &value)) { - failed = true; - } - tmp[j] = static_cast(value); - } break; - case KernelArg::Short: { - uint16_t* tmp = reinterpret_cast(globalData + offset); - uint value; - if (!getuintHex(source, &pos, &value)) { - failed = true; - } - tmp[j] = static_cast(value); - } break; - case KernelArg::Int: - case KernelArg::UInt: { - uint32_t* tmp = reinterpret_cast(globalData + offset); - if (!getuintHex(source, &pos, &tmp[j])) { - failed = true; - } - } break; - case KernelArg::Long: - case KernelArg::ULong: { - uint64_t* tmp = reinterpret_cast(globalData + offset); - if (!getuint64Hex(source, &pos, &tmp[j])) { - failed = true; - } - } break; - case KernelArg::NoType: - default: - break; - } - if (failed) { - delete[] globalData; - return false; - } - } - break; - } - } - if (posStart == pos) { - delete[] globalData; - return false; - } - posStart = pos = source.find_first_not_of(";# \n\r", pos); - } - - if (!allocGlobalData(globalData, dataSize, index)) { - failed = true; - } - - if (!clBinary()->storeGlobalData(globalData, dataSize, index)) { - failed = true; - } - - delete[] globalData; - - // Erase the global store information - if (index != 0) { - if (expect(source, &pos, ":")) { - // Read the size - if (!getuint(source, &pos, &index)) { - return false; - } - } - } - pos = source.find_first_not_of(";# \n\r", pos); - (const_cast(source)).erase(dataStart, pos - dataStart); - pos = dataStart; - if (failed) { - return false; - } - } - } - - return true; -} - -bool NullProgram::findILFuncs(const std::string& source, const std::string& func_start, - const std::string& func_end, size_t& lastFuncPos) { - lastFuncPos = 0; - - // Find first tag - size_t pos = source.find(func_start); - - // Loop through all provided program arguments - while (pos != std::string::npos) { - std::string funcName; - ILFunc func; - - func.code_.begin_ = pos; - if (!expect(source, &pos, func_start)) { - break; - } - - pos = source.find_first_not_of(" \n\r", pos); - // Read the function index - if (!getuint(source, &pos, &func.index_)) { - LogError("Error reading function index"); - return false; - } - - pos = source.find_first_of(";\n\r", pos); - if (source[pos] == '\r' || source[pos] == '\n') { - // this is the dummy macro - func.name_ = std::string(""); - } else { - pos = source.find_first_not_of("; \n\r", pos); - // Read the function's name - if (!getword(source, &pos, funcName)) { - LogError("Error reading function name"); - return false; - } - func.name_ = funcName; - } - - // Find the function end - pos = source.find(func_end, pos); - if (!expect(source, &pos, func_end)) { - break; - } - if (source[pos] == '\r' || source[pos] == '\n') { - if (!func.name_.empty()) { - LogError("Missing function name"); - return false; - } - } else { - // this is the dummy macro - pos = source.find_first_not_of("; \n\r", pos); - if (!expect(source, &pos, funcName)) { - LogError("Error reading function name"); - return false; - } - } - // Save the function end - func.code_.end_ = pos; - - if (!func.name_.empty()) { - // Create a new function - ILFunc* clFunc = new ILFunc(func); - if (clFunc != NULL) { - addFunc(clFunc); - } else { - return false; - } - } - lastFuncPos = pos; - // Next function - pos = source.find(func_start, pos); - } - - return true; -} - -bool NullProgram::findAllILFuncs(const std::string& source, size_t& lastFuncPos) { - // find all functions defined using "func" - size_t lastPos1; - bool ret = findILFuncs(source, "func ", "endfunc ", lastPos1); - if (!ret) return false; - - // find all functions defined using outlined macro - size_t lastPos2; - ret = findILFuncs(source, "mdef(", "mend", lastPos2); - if (!ret) return false; - - lastFuncPos = std::max(lastPos1, lastPos2); - return true; -} - -bool NullProgram::parseAllILFuncs(const std::string& source) { - bool doPatch = true; - amd::option::Options* opts = getCompilerOptions(); - if (opts->isCStrOptionsEqual(opts->oVariables->XLang, "il")) { - doPatch = false; - } - // Find the patch position - if (doPatch) { - patch_ = source.find(";$$$$$$$$$$"); - if (patch_ == std::string::npos) { - return false; - } - } - - size_t lastFuncPos = 0; - if (!findAllILFuncs(source, lastFuncPos)) { - return false; - } - - // Initialize the global data if available - if (!initGlobalData(source, lastFuncPos)) { - LogError("We failed the global constants detection/initialization!"); - return false; - } - - return true; -} - -bool NullProgram::parseFuncMetadata(const std::string& source, size_t posBegin, size_t posEnd) { - ILFunc* baseFunc = NULL; - uint index; - size_t pos = posBegin; - while (pos < posEnd) { - if (!expect(source, &pos, ";")) { - break; - } - for (uint k = 0; k < DescTotal; ++k) { - uint funcIndex; - uint j; - - if (expect(source, &pos, ArgState[k].typeName_)) { - if (ArgState[k].type_ == KernelArg::ErrorMessage) { - // Next argument - size_t posNext = source.find(";", pos); - buildLog_.append("Error:"); - buildLog_.append(source.substr(pos, posNext - pos)); - return false; - } else if (ArgState[k].type_ == KernelArg::WarningMessage) { - // Next argument - size_t posNext = source.find(";", pos); - buildLog_.append("Warning:"); - buildLog_.append(source.substr(pos, posNext - pos)); - continue; - } else if (ArgState[k].type_ == KernelArg::PrivateFixed) { - baseFunc->flags_ |= Kernel::PrivateFixed; - continue; - } else if (ArgState[k].type_ == KernelArg::ABI64Bit) { - baseFunc->flags_ |= Kernel::ABI64bit; - continue; - } else if (ArgState[k].type_ == KernelArg::Wavefront) { - baseFunc->flags_ |= Kernel::LimitWorkgroup; - continue; - } else if (ArgState[k].type_ == KernelArg::PrintfFormatStr) { - uint tmp; - uint arguments; - device::PrintfInfo info; - - // Read index - if (!getuint(source, &pos, &index)) { - return false; - } - if (printf_.size() <= index) { - printf_.resize(index + 1); - } - // Read the number of arguments - if (!getuint(source, &pos, &arguments)) { - return false; - } - for (uint j = 0; j < arguments; ++j) { - // Read the argument's size in bytes - if (!getuint(source, &pos, &tmp)) { - return false; - } - info.arguments_.push_back(tmp); - } - - // Read length - if (!getuint(source, &pos, &tmp)) { - return false; - } - // Read string (uses length so all possible chars are valid) - for (size_t i = 0; i < tmp; ++i) { - char symbol = source[pos++]; - if (symbol == '\\') { - // Rest of the C escape sequences (e.g. \') are handled correctly - // by the MDParser, we are not sure exactly how! - switch (source[pos]) { - case 'n': - pos++; - symbol = '\n'; - break; - case 'r': - pos++; - symbol = '\r'; - break; - case 'a': - pos++; - symbol = '\a'; - break; - case 'b': - pos++; - symbol = '\b'; - break; - case 'f': - pos++; - symbol = '\f'; - break; - case 'v': - pos++; - symbol = '\v'; - break; - default: - break; - } - } - info.fmtString_.push_back(symbol); - } - if (!expect(source, &pos, ";")) { - return false; - } - printf_[index] = info; - baseFunc->flags_ |= Kernel::PrintfOutput; - // Process next token ... - continue; - } else if (ArgState[k].type_ == KernelArg::MetadataVersion) { - continue; - } - - // Read the index - if (!getuint(source, &pos, &index)) { - return false; - } - - switch (ArgState[k].type_) { - case KernelArg::PrivateSize: - baseFunc->privateSize_ = index; - continue; - case KernelArg::LocalSize: - baseFunc->localSize_ = index; - continue; - case KernelArg::HwPrivateSize: - baseFunc->hwPrivateSize_ = index; - continue; - case KernelArg::HwLocalSize: - baseFunc->hwLocalSize_ = index; - continue; - default: - break; - } - - if (!ArgState[k].size_) { - // Find the base function - baseFunc = findILFunc(index); - if (baseFunc == NULL) { - return false; - } - // Sanity check - if (baseFunc->state_ != ILFunc::Unknown) { - buildLog_ = "Error: Creating kernel "; - buildLog_ += baseFunc->name_; - buildLog_ += " failed!\n"; - LogError(buildLog_.c_str()); - continue; - } - // If we have __OpenCL_ prefix in the name - // and _kernel suffix, then this is a kernel function - const std::string prefix = "__OpenCL_"; - const std::string postfix = "_kernel"; - const std::string& fname = baseFunc->name_; - size_t namelen = fname.size(); - size_t postfixPos = namelen - postfix.size(); - if (fname.compare(0, prefix.size(), prefix) == 0 && - fname.compare(postfixPos, namelen, postfix) == 0) { - baseFunc->state_ = ILFunc::Kernel; - baseFunc->name_.erase(postfixPos, postfix.size()); - baseFunc->name_.erase(0, prefix.size()); - } else { - baseFunc->state_ = ILFunc::Regular; - } - baseFunc->metadata_.begin_ = posBegin; - baseFunc->metadata_.end_ = posEnd; - continue; - } - - // Process metadata - for (j = 0; j < index; ++j) { - // Read the index - if (getuint(source, &pos, &funcIndex)) { - bool error = false; - if (ArgState[k].name_) { - ILFunc* func = findILFunc(funcIndex); - if (NULL != func) { - baseFunc->calls_.push_back(func); - } else { - buildLog_ += "Error: Undeclared function index "; - error = true; - } - } else { - if (funcIndex != 0xffffffff) { - baseFunc->macros_.push_back(funcIndex); - } else { - buildLog_ += "Error: Undeclared macro index "; - error = true; - } - } - if (error) { - char str[8]; - intToStr(funcIndex, str, 8); - buildLog_ += str; - buildLog_ += "\n"; - LogError("Undeclared index!"); - return false; - } - } else { - return false; - } - } - } - } - // Next argument - pos = source.find(";", pos); - } - return true; -} - -bool NullProgram::parseKernels(const std::string& source) { - size_t pos = 0; - - // Strip out all the debug tokens as these are - // not needed yet, but will be used later. - while (1) { - pos = source.find(";DEBUGSTART", pos); - if (pos == std::string::npos) { - break; - } - size_t last = source.find(";DEBUGEND", pos); - const_cast(source).erase(pos, last - pos + 10); - pos = last; - } - // Create a list of all functions in the program - if (!parseAllILFuncs(source)) { - return false; - } - pos = 0; - // Find all available metadata structures - for (size_t i = 0; i < funcs_.size(); ++i) { - std::string funcName; - ILFunc::SourceRange range; - - // Find function metadata start - range.begin_ = pos = source.find(";ARGSTART:", pos); - if (pos == std::string::npos) { - break; - } - - // Find function metadata end - pos = source.find(";ARGEND:", pos); - if (!expect(source, &pos, ";ARGEND:")) { - break; - } - // Read the function's name - if (!getword(source, &pos, funcName)) { - return false; - } - pos = source.find_first_not_of(" \n\r", pos); - range.end_ = pos; - if (!parseFuncMetadata(source, range.begin_, range.end_)) { - return false; - } - } - return true; -} - -void NullProgram::freeAllILFuncs() { - for (size_t i = 0; i < funcs_.size(); ++i) { - delete funcs_[i]; - } - funcs_.clear(); -} - -ILFunc* NullProgram::findILFunc(uint index) { - for (size_t i = 0; i < funcs_.size(); ++i) { - if (funcs_[i]->index_ == index) { - return funcs_[i]; - } - } - return NULL; -} - -NullKernel* NullProgram::createKernel(const std::string& name, const Kernel::InitData* initData, - const std::string& code, const std::string& metadata, - bool* created, const void* binaryCode, size_t binarySize) { - amd::option::Options* options = getCompilerOptions(); - uint64_t start_time = 0; - if (options->oVariables->EnableBuildTiming) { - start_time = amd::Os::timeNanos(); - } - - *created = false; - // Create a GPU kernel - NullKernel* gpuKernel = - new NullKernel(name, static_cast(device()), *this); - - if (gpuKernel == NULL) { - buildLog_ += "new Kernel() failed"; - LogPrintfError("new Kernel() failed for kernel %s!", name.c_str()); - return NULL; - } else if (gpuKernel->create(code, metadata, binaryCode, binarySize)) { - // Add kernel to the program - kernels()[gpuKernel->name()] = gpuKernel; - buildLog_ += gpuKernel->buildLog(); - } else { - buildError_ = gpuKernel->buildError(); - buildLog_ += gpuKernel->buildLog(); - delete gpuKernel; - LogPrintfError("Kernel creation failed for kernel %s!", name.c_str()); - return NULL; - } - - if (options->oVariables->EnableBuildTiming) { - std::stringstream tmp_ss; - tmp_ss << " Time for creating kernel (" << name - << ") : " << (amd::Os::timeNanos() - start_time) / 1000ULL << " us\n"; - buildLog_ += tmp_ss.str(); - } - - *created = true; - return gpuKernel; -} - -// Invoked from ClBinary -bool NullProgram::getAllKernelILs(std::unordered_map& allKernelILs, - std::string& programIL, const char* ilKernelName) { - llvm::CompUnit compunit(programIL); - if (ilKernelName != NULL) { - std::string MangeledName("__OpenCL_"); - MangeledName.append(ilKernelName); - MangeledName.append("_kernel"); - for (int i = 0; i < static_cast(compunit.getNumKernels()); ++i) { - std::string kernelname = compunit.getKernelName(i); - if (kernelname.compare(MangeledName) == 0) { - allKernelILs[kernelname] = compunit.getKernelStr(i); - break; - } - } - } else { - for (int i = 0; i < static_cast(compunit.getNumKernels()); ++i) { - std::string kernelname = compunit.getKernelName(i); - allKernelILs[kernelname] = compunit.getKernelStr(i); - } - } - return true; -} - -bool NullProgram::createBinary(amd::option::Options* options) { - if (options->oVariables->BinBIF30) { - return true; - } - - if (!clBinary()->createElfBinary(options->oVariables->BinEncrypt, type())) { - LogError("Failed to create ELF binary image!"); - return false; - } - return true; -} - -Program::~Program() { - // Destroy the global HW constant buffers - const Program::HwConstBuffers& gds = glbHwCb(); - for (const auto& it : gds) { - delete it.second; - } - - // Destroy the global data store - if (glbData_ != NULL) { - delete glbData_; - } -} - -bool Program::allocGlobalData(const void* globalData, size_t dataSize, uint index) { - bool result = false; - gpu::Memory* dataStore = NULL; - - if (index == 0) { - // We have to lock the heap block allocation, - // so possible reallocation won't occur twice or - // another thread could destroy a heap block, - // while we didn't finish allocation - amd::ScopedLock k(gpuDevice().lockAsyncOps()); - - // Allocate memory for the global data store - glbData_ = gpuDevice().createScratchBuffer(amd::alignUp(dataSize, 0x1000)); - dataStore = glbData_; - } else { - dataStore = new Memory(gpuDevice(), amd::alignUp(dataSize, ConstBuffer::VectorSize)); - - // Initialize constant buffer - if ((dataStore == NULL) || !dataStore->create(Resource::RemoteUSWC)) { - delete dataStore; - } else { - constBufs_[index] = dataStore; - glbCb_.push_back(index); - } - } - - if (dataStore != NULL) { - // Upload data to GPU memory - static const bool Entire = true; - amd::Coord3D origin(0, 0, 0); - amd::Coord3D region(dataSize); - result = gpuDevice().xferMgr().writeBuffer(globalData, *dataStore, origin, region, Entire); - } - - return result; -} - -bool Program::loadBinary(bool* hasRecompile) { - if (clBinary()->loadKernels(*this, hasRecompile)) { - // Load the global data - if (clBinary()->loadGlobalData(*this)) { - return true; - } - } - - // Make sure that kernels that have been generated so far shall be deleted. - clear(); - - return false; -} - -HSAILProgram::HSAILProgram(Device& device, amd::Program& owner) - : Program(device, owner), - rawBinary_(NULL), - kernels_(NULL), - maxScratchRegs_(0), - executable_(NULL), - loaderContext_(this) { - assert(device.isOnline()); - loader_ = amd::hsa::loader::Loader::Create(&loaderContext_); -} - -HSAILProgram::HSAILProgram(NullDevice& device, amd::Program& owner) - : Program(device, owner), - rawBinary_(NULL), - kernels_(NULL), - maxScratchRegs_(0), - executable_(NULL), - loaderContext_(this) { - assert(!device.isOnline()); - isNull_ = true; - - // Cannot load onto a NullDevice. - loader_ = nullptr; -} - -HSAILProgram::~HSAILProgram() { - // Destroy internal static samplers - for (auto& it : staticSamplers_) { - delete it; - } - if (rawBinary_ != NULL) { - amd::Hsail::FreeMem(binaryElf_, rawBinary_); - } - acl_error error; - // Free the elf binary - if (binaryElf_ != NULL) { - error = amd::Hsail::BinaryFini(binaryElf_); - if (error != ACL_SUCCESS) { - LogWarning("Error while destroying the acl binary \n"); - } - } - releaseClBinary(); - if (executable_ != NULL) { - loader_->DestroyExecutable(executable_); - } - delete kernels_; - amd::hsa::loader::Loader::Destroy(loader_); -} - -inline static std::vector splitSpaceSeparatedString(char* str) { - std::string s(str); - std::stringstream ss(s); - std::istream_iterator beg(ss), end; - std::vector vec(beg, end); - return vec; -} - -bool HSAILProgram::linkImpl(amd::option::Options* options) { - acl_error errorCode; - aclType continueCompileFrom = ACL_TYPE_LLVMIR_BINARY; - bool finalize = true; - bool hsaLoad = true; - // If !binaryElf_ then program must have been created using clCreateProgramWithBinary - if (!binaryElf_) { - continueCompileFrom = static_cast(getNextCompilationStageFromBinary(options)); - } - switch (continueCompileFrom) { - case ACL_TYPE_SPIRV_BINARY: - case ACL_TYPE_SPIR_BINARY: - // Compilation from ACL_TYPE_LLVMIR_BINARY to ACL_TYPE_CG in cases: - // 1. if the program is not created with binary; - // 2. if the program is created with binary and contains only .llvmir & .comment - // 3. if the program is created with binary, contains .llvmir, .comment, brig sections, - // but the binary's compile & link options differ from current ones (recompilation); - case ACL_TYPE_LLVMIR_BINARY: - // Compilation from ACL_TYPE_HSAIL_BINARY to ACL_TYPE_CG in cases: - // 1. if the program is created with binary and contains only brig sections - case ACL_TYPE_HSAIL_BINARY: - // Compilation from ACL_TYPE_HSAIL_TEXT to ACL_TYPE_CG in cases: - // 1. if the program is created with binary and contains only hsail text - case ACL_TYPE_HSAIL_TEXT: { - std::string curOptions = options->origOptionStr + hsailOptions(); - errorCode = amd::Hsail::Compile(gpuNullDevice().hsaCompiler(), binaryElf_, curOptions.c_str(), - continueCompileFrom, ACL_TYPE_CG, NULL); - buildLog_ += amd::Hsail::GetCompilerLog(gpuNullDevice().hsaCompiler()); - if (errorCode != ACL_SUCCESS) { - buildLog_ += "Error: BRIG code generation failed.\n"; - return false; - } - break; - } - case ACL_TYPE_CG: - break; - case ACL_TYPE_ISA: - finalize = false; - break; - default: - buildLog_ += - "Error: The binary is incorrect or incomplete. Finalization to ISA couldn't be " - "performed.\n"; - return false; - } - if (finalize) { - std::string fin_options(options->origOptionStr + hsailOptions()); - // Append an option so that we can selectively enable a SCOption on CZ - // whenever IOMMUv2 is enabled. - if (gpuNullDevice().settings().svmFineGrainSystem_) { - fin_options.append(" -sc-xnack-iommu"); - } - errorCode = amd::Hsail::Compile(gpuNullDevice().hsaCompiler(), binaryElf_, fin_options.c_str(), ACL_TYPE_CG, - ACL_TYPE_ISA, NULL); - buildLog_ += amd::Hsail::GetCompilerLog(gpuNullDevice().hsaCompiler()); - if (errorCode != ACL_SUCCESS) { - buildLog_ += "Error: BRIG finalization to ISA failed.\n"; - return false; - } - } - // ACL_TYPE_CG stage is not performed for offline compilation - hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; - if (hsaLoad) { - executable_ = loader_->CreateExecutable(HSA_PROFILE_FULL, NULL); - if (executable_ == NULL) { - buildLog_ += "Error: Executable for AMD HSA Code Object isn't created.\n"; - return false; - } - size_t size = 0; - hsa_code_object_t code_object; - code_object.handle = reinterpret_cast( - amd::Hsail::ExtractSection(gpuNullDevice().hsaCompiler(), binaryElf_, &size, aclTEXT, &errorCode)); - if (errorCode != ACL_SUCCESS) { - buildLog_ += "Error: Extracting AMD HSA Code Object from binary failed.\n"; - return false; - } - hsa_status_t status = executable_->LoadCodeObject(agent, code_object, NULL); - if (status != HSA_STATUS_SUCCESS) { - buildLog_ += "Error: AMD HSA Code Object loading failed.\n"; - return false; - } - } - size_t kernelNamesSize = 0; - errorCode = - amd::Hsail::QueryInfo(gpuNullDevice().hsaCompiler(), binaryElf_, RT_KERNEL_NAMES, NULL, NULL, &kernelNamesSize); - if (errorCode != ACL_SUCCESS) { - buildLog_ += "Error: Querying of kernel names size from the binary failed.\n"; - return false; - } - if (kernelNamesSize > 0) { - char* kernelNames = new char[kernelNamesSize]; - errorCode = amd::Hsail::QueryInfo(gpuNullDevice().hsaCompiler(), binaryElf_, RT_KERNEL_NAMES, NULL, kernelNames, - &kernelNamesSize); - if (errorCode != ACL_SUCCESS) { - buildLog_ += "Error: Querying of kernel names from the binary failed.\n"; - delete [] kernelNames; - return false; - } - std::vector vKernels = splitSpaceSeparatedString(kernelNames); - delete [] kernelNames; - bool dynamicParallelism = false; - aclMetadata md; - md.numHiddenKernelArgs = 0; - size_t sizeOfnumHiddenKernelArgs = sizeof(md.numHiddenKernelArgs); - for (const auto& it : vKernels) { - std::string kernelName(it); - std::string openclKernelName = Kernel::openclMangledName(kernelName); - errorCode = amd::Hsail::QueryInfo(gpuNullDevice().hsaCompiler(), binaryElf_, RT_NUM_KERNEL_HIDDEN_ARGS, - openclKernelName.c_str(), &md.numHiddenKernelArgs, - &sizeOfnumHiddenKernelArgs); - if (errorCode != ACL_SUCCESS) { - buildLog_ += "Error: Querying of kernel '" + openclKernelName + - "' extra arguments count from AMD HSA Code Object failed. Kernel initialization " - "failed.\n"; - return false; - } - HSAILKernel* aKernel = new HSAILKernel( - kernelName, this, options->origOptionStr + hsailOptions(), md.numHiddenKernelArgs); - kernels()[kernelName] = aKernel; - amd::hsa::loader::Symbol* sym = executable_->GetSymbol(openclKernelName.c_str(), &agent); - if (!sym) { - buildLog_ += "Error: Getting kernel ISA code symbol '" + openclKernelName + - "' from AMD HSA Code Object failed. Kernel initialization failed.\n"; - return false; - } - if (!aKernel->init(sym, false)) { - buildLog_ += "Error: Kernel '" + openclKernelName + "' initialization failed.\n"; - return false; - } - buildLog_ += aKernel->buildLog(); - aKernel->setUniformWorkGroupSize(options->oVariables->UniformWorkGroupSize); - dynamicParallelism |= aKernel->dynamicParallelism(); - // Find max scratch regs used in the program. It's used for scratch buffer preallocation - // with dynamic parallelism, since runtime doesn't know which child kernel will be called - maxScratchRegs_ = - std::max(static_cast(aKernel->workGroupInfo()->scratchRegs_), maxScratchRegs_); - } - // Allocate kernel table for device enqueuing - if (!isNull() && dynamicParallelism && !allocKernelTable()) { - return false; - } - } - // Save the binary in the interface class - saveBinaryAndSetType(TYPE_EXECUTABLE); - buildLog_ += amd::Hsail::GetCompilerLog(gpuNullDevice().hsaCompiler()); - return true; -} - -bool HSAILProgram::createBinary(amd::option::Options* options) { return true; } - -std::string HSAILProgram::hsailOptions() { - std::string hsailOptions; - // Set options for the standard device specific options - // All our devices support these options now - if (gpuNullDevice().settings().reportFMAF_) { - hsailOptions.append(" -DFP_FAST_FMAF=1"); - } - if (gpuNullDevice().settings().reportFMA_) { - hsailOptions.append(" -DFP_FAST_FMA=1"); - } - if (!gpuNullDevice().settings().singleFpDenorm_) { - hsailOptions.append(" -cl-denorms-are-zero"); - } - - // Check if the host is 64 bit or 32 bit - LP64_ONLY(hsailOptions.append(" -m64")); - - // Append each extension supported by the device - std::string token; - std::istringstream iss(""); - iss.str(device().info().extensions_); - while (getline(iss, token, ' ')) { - if (!token.empty()) { - hsailOptions.append(" -D"); - hsailOptions.append(token); - hsailOptions.append("=1"); - } - } - return hsailOptions; -} - -bool HSAILProgram::allocKernelTable() { - if (isNull()) { - // Cannot create a kernel table for offline devices. - return false; - } - - uint size = kernels().size() * sizeof(size_t); - - kernels_ = new gpu::Memory(gpuDevice(), size); - // Initialize kernel table - if ((kernels_ == NULL) || !kernels_->create(Resource::RemoteUSWC)) { - delete kernels_; - return false; - } else { - size_t* table = reinterpret_cast(kernels_->map(NULL, gpu::Resource::WriteOnly)); - for (auto& it : kernels()) { - HSAILKernel* kernel = static_cast(it.second); - table[kernel->index()] = static_cast(kernel->gpuAqlCode()->vmAddress()); - } - kernels_->unmap(NULL); - } - return true; -} - -void HSAILProgram::fillResListWithKernels(std::vector& memList) const { - for (auto& it : kernels()) { - memList.push_back(static_cast(it.second)->gpuAqlCode()); - } -} - -const aclTargetInfo& HSAILProgram::info() { - acl_error err; - info_ = amd::Hsail::GetTargetInfo(gpuNullDevice().settings().use64BitPtr_ ? "hsail64" : "hsail", - device().isa().hsailName(), &err); - if (err != ACL_SUCCESS) { - LogWarning("aclGetTargetInfo failed"); - } - return info_; -} - -bool HSAILProgram::saveBinaryAndSetType(type_t type) { - // Write binary to memory - if (rawBinary_ != NULL) { - // Free memory containing rawBinary - amd::Hsail::FreeMem(binaryElf_, rawBinary_); - rawBinary_ = NULL; - } - size_t size = 0; - if (amd::Hsail::WriteToMem(binaryElf_, &rawBinary_, &size) != ACL_SUCCESS) { - buildLog_ += "Failed to write binary to memory \n"; - return false; - } - setBinary(static_cast(rawBinary_), size); - // Set the type of binary - setType(type); - return true; -} - -hsa_isa_t ORCAHSALoaderContext::IsaFromName(const char* name) { - const amd::Isa* isa_p = amd::Isa::findIsa(name); - return {amd::Isa::toHandle(isa_p)}; -} - -bool ORCAHSALoaderContext::IsaSupportedByAgent(hsa_agent_t agent, hsa_isa_t isa) { - // The HSA loader uses a handle value of 0 to indicate the ISA is invalid. - const amd::Isa* code_object_isa_p = amd::Isa::fromHandle(isa.handle); - if (!code_object_isa_p || !code_object_isa_p->runtimeGslSupported()) { - // The ISA is either not supported because ORCAHSALoaderContext::IsaFromName - // could not find it, or the PAL runtime does not support it. - return false; - } - if (program_->isNull()) { - // Cannot load code onto offline devices. - return false; - } - return amd::Isa::isCompatible(*code_object_isa_p, program_->device().isa()); -} - -void* ORCAHSALoaderContext::SegmentAlloc(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, - size_t size, size_t align, bool zero) { - assert(size); - assert(align); - switch (segment) { - case AMDGPU_HSA_SEGMENT_GLOBAL_PROGRAM: - case AMDGPU_HSA_SEGMENT_GLOBAL_AGENT: - case AMDGPU_HSA_SEGMENT_READONLY_AGENT: - return AgentGlobalAlloc(agent, size, align, zero); - case AMDGPU_HSA_SEGMENT_CODE_AGENT: - return KernelCodeAlloc(agent, size, align, zero); - default: - assert(false); - return 0; - } -} - -bool ORCAHSALoaderContext::SegmentCopy(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, - void* dst, size_t offset, const void* src, size_t size) { - switch (segment) { - case AMDGPU_HSA_SEGMENT_GLOBAL_PROGRAM: - case AMDGPU_HSA_SEGMENT_GLOBAL_AGENT: - case AMDGPU_HSA_SEGMENT_READONLY_AGENT: - return AgentGlobalCopy(dst, offset, src, size); - case AMDGPU_HSA_SEGMENT_CODE_AGENT: - return KernelCodeCopy(dst, offset, src, size); - default: - assert(false); - return false; - } -} - -void ORCAHSALoaderContext::SegmentFree(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, - void* seg, size_t size) { - switch (segment) { - case AMDGPU_HSA_SEGMENT_GLOBAL_PROGRAM: - case AMDGPU_HSA_SEGMENT_GLOBAL_AGENT: - case AMDGPU_HSA_SEGMENT_READONLY_AGENT: - AgentGlobalFree(seg, size); - break; - case AMDGPU_HSA_SEGMENT_CODE_AGENT: - KernelCodeFree(seg, size); - break; - default: - assert(false); - return; - } -} - -void* ORCAHSALoaderContext::SegmentAddress(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, - void* seg, size_t offset) { - assert(seg); - switch (segment) { - case AMDGPU_HSA_SEGMENT_GLOBAL_PROGRAM: - case AMDGPU_HSA_SEGMENT_GLOBAL_AGENT: - case AMDGPU_HSA_SEGMENT_READONLY_AGENT: { - if (!program_->isNull()) { - gpu::Memory* gpuMem = reinterpret_cast(seg); - return reinterpret_cast(gpuMem->vmAddress() + offset); - } - } - case AMDGPU_HSA_SEGMENT_CODE_AGENT: - return (char*)seg + offset; - default: - assert(false); - return NULL; - } -} - -hsa_status_t ORCAHSALoaderContext::SamplerCreate( - hsa_agent_t agent, const hsa_ext_sampler_descriptor_t* sampler_descriptor, - hsa_ext_sampler_t* sampler_handle) { - sampler_handle->handle = 0; - if (!agent.handle) { - return HSA_STATUS_ERROR_INVALID_AGENT; - } - if (!sampler_descriptor || !sampler_handle) { - return HSA_STATUS_ERROR_INVALID_ARGUMENT; - } - - if (program_->isNull()) { - // Offline compilation. Provide a fake non-null handle. - sampler_handle->handle = 1; - return HSA_STATUS_SUCCESS; - } - - uint32_t state = 0; - switch (sampler_descriptor->coordinate_mode) { - case HSA_EXT_SAMPLER_COORDINATE_MODE_UNNORMALIZED: - state = amd::Sampler::StateNormalizedCoordsFalse; - break; - case HSA_EXT_SAMPLER_COORDINATE_MODE_NORMALIZED: - state = amd::Sampler::StateNormalizedCoordsTrue; - break; - default: - assert(false); - return HSA_STATUS_ERROR_INVALID_ARGUMENT; - } - switch (sampler_descriptor->filter_mode) { - case HSA_EXT_SAMPLER_FILTER_MODE_NEAREST: - state |= amd::Sampler::StateFilterNearest; - break; - case HSA_EXT_SAMPLER_FILTER_MODE_LINEAR: - state |= amd::Sampler::StateFilterLinear; - break; - default: - assert(false); - return HSA_STATUS_ERROR_INVALID_ARGUMENT; - } - switch (sampler_descriptor->address_mode) { - case HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE: - state |= amd::Sampler::StateAddressClampToEdge; - break; - case HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_BORDER: - state |= amd::Sampler::StateAddressClamp; - break; - case HSA_EXT_SAMPLER_ADDRESSING_MODE_REPEAT: - state |= amd::Sampler::StateAddressRepeat; - break; - case HSA_EXT_SAMPLER_ADDRESSING_MODE_MIRRORED_REPEAT: - state |= amd::Sampler::StateAddressMirroredRepeat; - break; - case HSA_EXT_SAMPLER_ADDRESSING_MODE_UNDEFINED: - state |= amd::Sampler::StateAddressNone; - break; - default: - assert(false); - return HSA_STATUS_ERROR_INVALID_ARGUMENT; - } - std::unique_ptr sampler(new gpu::Sampler(program_->gpuDevice())); - if (!sampler || !sampler->create(state)) { - return HSA_STATUS_ERROR; - } - sampler_handle->handle = sampler->hwSrd(); - program_->addSampler(sampler.release()); - return HSA_STATUS_SUCCESS; -} - -hsa_status_t ORCAHSALoaderContext::SamplerDestroy(hsa_agent_t agent, - hsa_ext_sampler_t sampler_handle) { - if (!agent.handle) { - return HSA_STATUS_ERROR_INVALID_AGENT; - } - if (!sampler_handle.handle) { - return HSA_STATUS_ERROR_INVALID_ARGUMENT; - } - // Samplers will be destroyed by the pal::HSAILProgam destructor. - return HSA_STATUS_SUCCESS; -} - -void* ORCAHSALoaderContext::CpuMemAlloc(size_t size, size_t align, bool zero) { - assert(size); - assert(align); - assert(sizeof(void*) == 8 || sizeof(void*) == 4); - void* ptr = amd::Os::alignedMalloc(size, align); - if (zero) { - memset(ptr, 0, size); - } - return ptr; -} - -bool ORCAHSALoaderContext::CpuMemCopy(void* dst, size_t offset, const void* src, size_t size) { - if (!dst || !src || dst == src) { - return false; - } - if (0 == size) { - return true; - } - amd::Os::fastMemcpy((char*)dst + offset, src, size); - return true; -} - -void* ORCAHSALoaderContext::GpuMemAlloc(size_t size, size_t align, bool zero) { - assert(size); - assert(align); - assert(sizeof(void*) == 8 || sizeof(void*) == 4); - if (program_->isNull()) { - return new char[size]; - } - - gpu::Memory* mem = new gpu::Memory(program_->gpuDevice(), amd::alignUp(size, align)); - if (!mem || !mem->create(gpu::Resource::Local)) { - delete mem; - return NULL; - } - assert(program_->gpuDevice().xferQueue()); - if (zero) { - char pattern = 0; - program_->gpuDevice().xferMgr().fillBuffer(*mem, &pattern, sizeof(pattern), amd::Coord3D(0), - amd::Coord3D(size)); - } - program_->addGlobalStore(mem); - program_->setGlobalVariableTotalSize(program_->globalVariableTotalSize() + size); - return mem; -} - -bool ORCAHSALoaderContext::GpuMemCopy(void* dst, size_t offset, const void* src, size_t size) { - if (!dst || !src || dst == src) { - return false; - } - if (0 == size) { - return true; - } - if (program_->isNull()) { - memcpy(reinterpret_cast
(dst) + offset, src, size); - return true; - } - assert(program_->gpuDevice().xferQueue()); - gpu::Memory* mem = reinterpret_cast(dst); - return program_->gpuDevice().xferMgr().writeBuffer(src, *mem, amd::Coord3D(offset), amd::Coord3D(size), - true); -} - -void ORCAHSALoaderContext::GpuMemFree(void* ptr, size_t size) { - if (program_->isNull()) { - delete[] reinterpret_cast(ptr); - } else { - delete reinterpret_cast(ptr); - } -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuprogram.hpp b/projects/clr/rocclr/device/gpu/gpuprogram.hpp deleted file mode 100644 index fe7026df4a..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuprogram.hpp +++ /dev/null @@ -1,535 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUPROGRAM_HPP_ -#define GPUPROGRAM_HPP_ - -#include "device/gpu/gpukernel.hpp" -#include "device/gpu/gpubinary.hpp" -#include "amd_hsa_loader.hpp" - -namespace amd { -namespace option { -class Options; -} // option -namespace hsa { -namespace loader { -class Loader; -class Executable; -class Context; -} // loader -} // hsa -} // amd - -//! \namespace gpu GPU Device Implementation -namespace gpu { - -/*! \addtogroup GPU GPU Device Implementation - * @{ - */ - -//! \struct ILFunc for the opencl program processing -struct ILFunc : public amd::HeapObject { - public: - //! \struct CodeRange for the code ranges - struct SourceRange : public amd::EmbeddedObject { - size_t begin_; //!< start code position - size_t end_; //!< end code position - }; - - //! \enum IL function state - enum State { - Unknown = 0x00000000, //! unknown function - Regular = 0x00000001, //! regular function from the program - Kernel = 0x00000002 //! kernel function from the program - }; - - //! Default constructor - ILFunc() - : name_(""), - index_(0), - state_(Unknown), - privateSize_(0), - localSize_(0), - hwPrivateSize_(0), - hwLocalSize_(0), - flags_(0), - totalHwPrivateSize_(-1) { - code_.begin_ = code_.end_ = 0; - metadata_.begin_ = metadata_.end_ = 0; - } - - //! Copy constructor - ILFunc(const ILFunc& func) { *this = func; } - - //! Destructor - ~ILFunc() {} - - //! Overloads operator= - ILFunc& operator=(const ILFunc& func) { - name_ = func.name_; - index_ = func.index_; - code_ = func.code_; - metadata_ = func.metadata_; - state_ = func.state_; - privateSize_ = func.privateSize_; - localSize_ = func.localSize_; - hwPrivateSize_ = func.hwPrivateSize_; - hwLocalSize_ = func.hwLocalSize_; - flags_ = func.flags_; - totalHwPrivateSize_ = func.totalHwPrivateSize_; - - // Note: we don't copy calls_ and macros_ - return *this; - } - - std::string name_; //!< kernel's name - uint index_; //!< kernel's index - SourceRange code_; //!< the entire function range in the source - SourceRange metadata_; //!< the metadata range - State state_; //!< the function is real, and not intrinsic - uint privateSize_; //!< private ring allocation by the function - uint localSize_; //!< local ring allocation by the function - uint hwPrivateSize_; //!< HW private ring allocation by the function - uint hwLocalSize_; //!< HW local ring allocation by the function - uint flags_; //!< The IL func flags/properties - long long totalHwPrivateSize_; //!< total HW private usage including called functions - std::vector calls_; //! Functions called from the current - std::vector macros_; //! Macros, used in the IL function - - uint totalHwPrivateUsage(); //!< total HW private usage including called functions -}; - -//! \class empty program -class NullProgram : public device::Program { - friend class ClBinary; - - public: - //! Default constructor - NullProgram(NullDevice& nullDev, amd::Program& owner) - : device::Program(nullDev, owner), patch_(0) {} - - //! Default destructor - ~NullProgram(); - - // Initialize Binary for GPU - virtual bool initClBinary(); - - //! Returns global constant buffers - const std::vector& glbCb() const { return glbCb_; } - - protected: - /*! \brief Compiles GPU CL program to LLVM binary (compiler frontend) - * - * \return True if we successfully compiled a GPU program - */ - virtual bool compileImpl(const std::string& sourceCode, //!< the program's source code - const std::vector& headers, //!< header souce codes - const char** headerIncludeNames, //!< include names of headers - amd::option::Options* options //!< compile options's object - ); - - /*! \brief Compiles LLVM binary to IL code (compiler backend: link+opt+codegen) - * - * \return The build error code - */ - int compileBinaryToIL(amd::option::Options* options //!< options for compilation - ); - - /*! \brief Links the compiled IL program with HW - * - * \return True if we successefully linked a GPU program - */ - virtual bool linkImpl(amd::option::Options* options = NULL //!< options object - ); - virtual bool linkImpl(const std::vector& inputPrograms, - amd::option::Options* options = NULL, //!< options object - bool createLibrary = false); - - virtual bool createBinary(amd::option::Options* options); - - - /*! \brief Parses the GPU program and finds all available kernels - * - * \return True if we successfully parsed the GPU program - */ - bool parseKernels(const std::string& source //! the program's source code - ); - - /*! \brief Parse all functions in the program - * - * \return True if we successfully parsed all functions - */ - bool parseAllILFuncs(const std::string& source //! the program's source code - ); - - /*! \brief Parse a function's metadata given as source[posBegin:posEnd-1] - * - * \return True if we successfully parsed the given metadata - */ - bool parseFuncMetadata(const std::string& source, //! string that contains metadata - size_t posBegin, //! begin of metadata in 'source' - size_t posEnd //! end of metadata in 'source' - ); - - /*! \brief Finds functions with the given start and end string in the - * program - * - * \return True if we successfully found all functions - */ - bool findILFuncs(const std::string& source, //! the program's source code - const std::string& func_start, //! the start string of a function - const std::string& func_end, //! the end string of a function - size_t& lastFuncPos //! pos to the end of the last func in 'source' - ); - - - /*! \brief Finds all functions in the program - * - * \return True if we successefully found all functions - */ - bool findAllILFuncs(const std::string& source, //! the program's source code - size_t& lastFuncPos //! pos to the end of the last func in 'source' - ); - - /*! \brief Finds function, corresponded to the provided unique index - * - * \return Pointer to the ILFunc structure - */ - ILFunc* findILFunc(uint index //! the function unique index - ); - - //! Destroys all objects, associated with the IL functions - void freeAllILFuncs(); - - /*! \brief Finds if a provided function is called from the base function - * - * \return True if a function is used from the base one - */ - bool isCalled(const ILFunc* base, //!< The base function - const ILFunc* func //!< Function to check for usage - ); - - //! Patches the "main" function with the call to the current kernel - void patchMain(std::string& kernel, //! The current kernel's code for compilation - uint index //! Index of the current kernel in the program - ); - - //! Adds the IL function object into the list of functions - void addFunc(ILFunc* func) { funcs_.push_back(func); } - - //! Empty implementation, since we don't have real HW - virtual bool allocGlobalData(const void* globalData, //!< Pointer to the global data - size_t dataSize, //!< The global data size - uint index //!< Index for the global data store (0 - global heap) - ) { - glbCb_.push_back(index); - return true; - } - - //! Load binary for offline device. - virtual bool loadBinary(bool* hasRecompiled); - - //! Create NullKernel for compiling to isa. - virtual NullKernel* createKernel(const std::string& name, //!< The kernel's name - const Kernel::InitData* initData, //!< Initialization data - const std::string& code, //!< IL source code - const std::string& metadata, //!< the kernel metadata structure - bool* created, //!< True if the object was created - const void* binaryCode = NULL, //!< binary machine code for CAL - size_t binarySize = 0 //!< the machine code size - ); - - ClBinary* clBinary() { return static_cast(device::Program::clBinary()); } - const ClBinary* clBinary() const { - return static_cast(device::Program::clBinary()); - } - - /*! Get all per-kernel IL from programIL, where programIL is the IL for the - * whole compilation unit. - */ - bool getAllKernelILs(std::unordered_map& allKernelILs, std::string& programIL, - const char* ilKernelName); - - protected: - std::vector printf_; //!< Format strings for GPU printf support - std::vector glbCb_; //!< Global constant buffers - - virtual const aclTargetInfo& info(); - - virtual bool saveBinaryAndSetType(type_t type) { return true; } - - private: - //! Disable default copy constructor - NullProgram(const NullProgram&); - - //! Disable operator= - NullProgram& operator=(const NullProgram&); - - //! Initializes the global data store - bool initGlobalData(const std::string& source, //!< the program's source code - size_t start //!< start position for the global data search - ); - - //! Return a typecasted GPU device - gpu::NullDevice& gpuNullDevice() { - return const_cast(static_cast(device())); - } - - size_t patch_; //!< Patch call position in the source code. - std::vector funcs_; //!< list of all functions. - - std::string ilProgram_; //!< IL program after compilation -}; - -//! \class GPU program -class Program : public NullProgram { - public: - //! GPU program constructor - Program(Device& gpuDev, amd::Program& owner) : NullProgram(gpuDev, owner), glbData_(NULL) {} - - //! GPU program destructor - ~Program(); - - //! Get the global data store for this program - gpu::Memory* glbData() const { return glbData_; } - - //! Returns TRUE if we successfully allocated the global data store - //! in video memory - bool allocGlobalData(const void* globalData, //!< Pointer to the global data - size_t dataSize, //!< The global data size - uint index //!< Index for the global data store (0 - global heap) - ); - - //! Returns TRUE if we could - virtual bool loadBinary(bool* hasRecompiled); - - //! Creates the GPU kernel (return base type) - virtual NullKernel* createKernel(const std::string& name, //!< The kernel's name - const Kernel::InitData* initData, //!< Initialization data - const std::string& code, //!< IL source code - const std::string& metadata, //!< the kernel metadata structure - bool* created, //!< True if the object was created - const void* binaryCode = NULL, //!< binary machine code for CAL - size_t binarySize = 0 //!< the machine code size - ); - - typedef std::unordered_map HwConstBuffers; - - //! Global HW constant buffers - const HwConstBuffers& glbHwCb() const { return constBufs_; } - - //! Returns pritnf info array - const std::vector& printfInfo() const { return printf_; } - - protected: - private: - //! Disable copy constructor - Program(const Program&); - - //! Disable operator= - Program& operator=(const Program&); - - //! Return a typecasted GPU device - gpu::Device& gpuDevice() { - assert(!isNull()); - return const_cast(static_cast(device())); - } - - HwConstBuffers constBufs_; //!< Constant buffers for the global store - gpu::Memory* glbData_; //!< Global data store -}; - -using namespace amd::hsa::loader; -class HSAILProgram; - -class ORCAHSALoaderContext final : public Context { - public: - ORCAHSALoaderContext(HSAILProgram* program) : program_(program) {} - - virtual ~ORCAHSALoaderContext() {} - - hsa_isa_t IsaFromName(const char* name) override; - - bool IsaSupportedByAgent(hsa_agent_t agent, hsa_isa_t isa) override; - - void* SegmentAlloc(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, size_t size, size_t align, - bool zero) override; - - bool SegmentCopy(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, void* dst, size_t offset, - const void* src, size_t size) override; - - void SegmentFree(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, void* seg, - size_t size = 0) override; - - void* SegmentAddress(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, void* seg, - size_t offset) override; - - void* SegmentHostAddress(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, void* seg, - size_t offset) override { - return nullptr; - } - - bool SegmentFreeze(amdgpu_hsa_elf_segment_t segment, hsa_agent_t agent, void* seg, - size_t size) override { - return false; - } - - bool ImageExtensionSupported() override { return false; } - - hsa_status_t ImageCreate(hsa_agent_t agent, hsa_access_permission_t image_permission, - const hsa_ext_image_descriptor_t* image_descriptor, - const void* image_data, hsa_ext_image_t* image_handle) override { - // not supported - assert(false); - return HSA_STATUS_ERROR; - } - - hsa_status_t ImageDestroy(hsa_agent_t agent, hsa_ext_image_t image_handle) override { - // not supported - assert(false); - return HSA_STATUS_ERROR; - } - - hsa_status_t SamplerCreate(hsa_agent_t agent, - const hsa_ext_sampler_descriptor_t* sampler_descriptor, - hsa_ext_sampler_t* sampler_handle) override; - - //! All samplers are owned by HSAILProgram and are deleted in its destructor. - hsa_status_t SamplerDestroy(hsa_agent_t agent, hsa_ext_sampler_t sampler_handle) override; - - private: - void* AgentGlobalAlloc(hsa_agent_t agent, size_t size, size_t align, bool zero) { - return GpuMemAlloc(size, align, zero); - } - - bool AgentGlobalCopy(void* dst, size_t offset, const void* src, size_t size) { - return GpuMemCopy(dst, offset, src, size); - } - - void AgentGlobalFree(void* ptr, size_t size) { GpuMemFree(ptr, size); } - - void* KernelCodeAlloc(hsa_agent_t agent, size_t size, size_t align, bool zero) { - return CpuMemAlloc(size, align, zero); - } - - bool KernelCodeCopy(void* dst, size_t offset, const void* src, size_t size) { - return CpuMemCopy(dst, offset, src, size); - } - - void KernelCodeFree(void* ptr, size_t size) { CpuMemFree(ptr, size); } - - void* CpuMemAlloc(size_t size, size_t align, bool zero); - - bool CpuMemCopy(void* dst, size_t offset, const void* src, size_t size); - - void CpuMemFree(void* ptr, size_t size) { amd::Os::alignedFree(ptr); } - - void* GpuMemAlloc(size_t size, size_t align, bool zero); - - bool GpuMemCopy(void* dst, size_t offset, const void* src, size_t size); - - void GpuMemFree(void* ptr, size_t size = 0); - - ORCAHSALoaderContext(const ORCAHSALoaderContext& c); - - ORCAHSALoaderContext& operator=(const ORCAHSALoaderContext& c); - - gpu::HSAILProgram* program_; -}; - -//! \class HSAIL program -class HSAILProgram : public device::Program { - friend class ClBinary; - - public: - //! Default constructor - HSAILProgram(Device& device, amd::Program& owner); - HSAILProgram(NullDevice& device, amd::Program& owner); - //! Default destructor - ~HSAILProgram(); - - void addGlobalStore(Memory* mem) { globalStores_.push_back(mem); } - - const std::vector& globalStores() const { return globalStores_; } - - //! Return a typecasted GPU device. The device must not be the NullDevice. - gpu::Device& gpuDevice() { - assert(!isNull()); - return const_cast(static_cast(device())); - } - - //! Returns GPU kernel table - const Memory* kernelTable() const { return kernels_; } - - //! Adds all kernels to the mem handle lists - void fillResListWithKernels(std::vector& memList) const; - - //! Returns the maximum number of scratch regs used in the program - uint maxScratchRegs() const { return maxScratchRegs_; } - - //! Add internal static sampler - void addSampler(Sampler* sampler) { staticSamplers_.push_back(sampler); } - - //! Returns TRUE if the program contains static samplers - bool isStaticSampler() const { return (staticSamplers_.size() != 0); } - - protected: - bool saveBinaryAndSetType(type_t type); - - virtual bool linkImpl(amd::option::Options* options); - - virtual bool createBinary(amd::option::Options* options); - - virtual const aclTargetInfo& info(); - - private: - //! Disable default copy constructor - HSAILProgram(const HSAILProgram&); - - //! Disable operator= - HSAILProgram& operator=(const HSAILProgram&); - - //! Returns all the options to be appended while passing to the - // compiler library - std::string hsailOptions(); - - //! Allocate kernel table - bool allocKernelTable(); - - //! Return a typecasted GPU null device. - gpu::NullDevice& gpuNullDevice() { - return const_cast(static_cast(device())); - } - - void* rawBinary_; //!< Pointer to the raw binary - std::vector globalStores_; //!< Global memory for the program - Memory* kernels_; //!< Table with kernel object pointers - uint - maxScratchRegs_; //!< Maximum number of scratch regs used in the program by individual kernel - std::list staticSamplers_; //!< List od internal static samplers - amd::hsa::loader::Loader* loader_; //!< Loader object - amd::hsa::loader::Executable* executable_; //!< Executable for HSA Loader - ORCAHSALoaderContext loaderContext_; //!< Context for HSA Loader -}; - -/*@}*/} // namespace gpu - -#endif /*GPUPROGRAM_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpuresource.cpp b/projects/clr/rocclr/device/gpu/gpuresource.cpp deleted file mode 100644 index 96b153a88f..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuresource.cpp +++ /dev/null @@ -1,1858 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/device.hpp" - -#if defined(ATI_OS_WIN) -#define WIN32_LEAN_AND_MEAN 1 -#include -#endif -#include -#include "GL/glATIInternal.h" - -#include "os/os.hpp" -#include "utils/flags.hpp" -#include "thread/monitor.hpp" -#include "device/gpu/gpuresource.hpp" -#include "device/gpu/gpudevice.hpp" -#include "device/gpu/gpublit.hpp" -#include "device/gpu/gputimestamp.hpp" -#include "hsa_ext_image.h" - -#include -#include -#include -#include -#include - -namespace gpu { - -GslResourceReference::GslResourceReference(const Device& gpuDev, gslMemObject gslResource, - gslMemObject gslResOriginal) - : device_(gpuDev), resource_(gslResource), resOriginal_(gslResOriginal), cpuAddress_(NULL) {} - -GslResourceReference::~GslResourceReference() { - if (cpuAddress_ != NULL) { - device_.resUnmapRemote(gslResource()); - } - if (0 != gslResource()) { - device_.resFree(gslResource()); - resource_ = NULL; - } - - if (0 != gslOriginal()) { - device_.resFree(gslOriginal()); - resOriginal_ = NULL; - } -} - -Resource::Resource(const Device& gpuDev, size_t width, cmSurfFmt format) - : elementSize_(0), - gpuDevice_(gpuDev), - mapCount_(0), - address_(NULL), - offset_(0), - curRename_(0), - gslRef_(NULL), - viewOwner_(NULL), - hbOffset_(0), - hbSize_(0), - pinOffset_(0), - glInterop_(0), - gpu_(NULL) { - // Fill GSL descriptor fields - cal_.type_ = Empty; - cal_.width_ = width; - cal_.height_ = 1; - cal_.depth_ = 1; - cal_.mipLevels_ = 1; - cal_.format_ = format; - cal_.flags_ = 0; - cal_.pitch_ = 0; - cal_.slice_ = 0; - cal_.channelOrder_ = GSL_CHANNEL_ORDER_REPLICATE_R; - cal_.dimension_ = GSL_MOA_BUFFER; - cal_.cardMemory_ = true; - cal_.dimSize_ = 1; - cal_.buffer_ = true; - cal_.imageArray_ = false; - cal_.imageType_ = 0; - cal_.skipRsrcCache_ = false; - cal_.scratch_ = false; - cal_.isAllocSVM_ = false; - cal_.isAllocExecute_ = false; -} - -Resource::Resource(const Device& gpuDev, size_t width, size_t height, size_t depth, - cmSurfFmt format, gslChannelOrder chOrder, cl_mem_object_type imageType, - uint mipLevels) - : elementSize_(0), - gpuDevice_(gpuDev), - mapCount_(0), - address_(NULL), - offset_(0), - curRename_(0), - gslRef_(NULL), - viewOwner_(NULL), - hbOffset_(0), - hbSize_(0), - pinOffset_(0), - glInterop_(0), - gpu_(NULL) { - // Fill GSL descriptor fields - cal_.type_ = Empty; - cal_.width_ = width; - cal_.height_ = height; - cal_.depth_ = depth; - cal_.mipLevels_ = mipLevels; - cal_.format_ = format; - cal_.flags_ = 0; - cal_.pitch_ = 0; - cal_.slice_ = 0; - cal_.channelOrder_ = chOrder; - cal_.cardMemory_ = true; - cal_.buffer_ = false; - cal_.imageArray_ = false; - cal_.imageType_ = imageType; - cal_.skipRsrcCache_ = false; - cal_.scratch_ = false; - cal_.isAllocSVM_ = false; - cal_.isAllocExecute_ = false; - - switch (imageType) { - case CL_MEM_OBJECT_IMAGE2D: - cal_.dimension_ = GSL_MOA_TEXTURE_2D; - cal_.dimSize_ = 2; - break; - case CL_MEM_OBJECT_IMAGE3D: - cal_.dimension_ = GSL_MOA_TEXTURE_3D; - cal_.dimSize_ = 3; - break; - case CL_MEM_OBJECT_IMAGE2D_ARRAY: - cal_.dimension_ = GSL_MOA_TEXTURE_2D_ARRAY; - cal_.dimSize_ = 3; - cal_.imageArray_ = true; - break; - case CL_MEM_OBJECT_IMAGE1D: - cal_.dimension_ = GSL_MOA_TEXTURE_1D; - cal_.dimSize_ = 1; - break; - case CL_MEM_OBJECT_IMAGE1D_ARRAY: - cal_.dimension_ = GSL_MOA_TEXTURE_1D_ARRAY; - cal_.dimSize_ = 2; - cal_.imageArray_ = true; - break; - case CL_MEM_OBJECT_IMAGE1D_BUFFER: - cal_.dimension_ = GSL_MOA_TEXTURE_BUFFER; - cal_.dimSize_ = 1; - break; - default: - cal_.dimSize_ = 1; - LogError("Unknown image type!"); - break; - } -} - -Resource::~Resource() { free(); } - -static uint32_t GetHSAILImageFormatType(cmSurfFmt format) { - uint32_t formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT8; - - switch (format) { - case CM_SURF_FMT_sR8: - case CM_SURF_FMT_sRG8: - case CM_SURF_FMT_sRGBA8: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT8; - break; - case CM_SURF_FMT_sU16: - case CM_SURF_FMT_sUV16: - case CM_SURF_FMT_sUVWQ16: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_SNORM_INT16; - break; - case CM_SURF_FMT_INTENSITY8: - case CM_SURF_FMT_RG8: - case CM_SURF_FMT_RGBA8: - case CM_SURF_FMT_RGBX8UI: - case CM_SURF_FMT_RGBA8_SRGB: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT8; - break; - case CM_SURF_FMT_R16: - case CM_SURF_FMT_RG16: - case CM_SURF_FMT_RGBA16: - case CM_SURF_FMT_DEPTH16: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT16; - break; - case CM_SURF_FMT_BGR10_X2: - case CM_SURF_FMT_RGB10_X2: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_SHORT_101010; - break; - case CM_SURF_FMT_sR8I: - case CM_SURF_FMT_sRG8I: - case CM_SURF_FMT_sRGBA8I: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT8; - break; - case CM_SURF_FMT_sR16I: - case CM_SURF_FMT_sRG16I: - case CM_SURF_FMT_sRGBA16I: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT16; - break; - case CM_SURF_FMT_sR32I: - case CM_SURF_FMT_sRG32I: - case CM_SURF_FMT_sRGBA32I: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_SIGNED_INT32; - break; - case CM_SURF_FMT_R8I: - case CM_SURF_FMT_RG8I: - case CM_SURF_FMT_RGBA8UI: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8; - break; - case CM_SURF_FMT_R16I: - case CM_SURF_FMT_RG16I: - case CM_SURF_FMT_RGBA16UI: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16; - break; - case CM_SURF_FMT_R32I: - case CM_SURF_FMT_RG32I: - case CM_SURF_FMT_RGBA32UI: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32; - break; - case CM_SURF_FMT_R16F: - case CM_SURF_FMT_RG16F: - case CM_SURF_FMT_RGBA16F: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_HALF_FLOAT; - break; - case CM_SURF_FMT_R32F: - case CM_SURF_FMT_RG32F: - case CM_SURF_FMT_RGBA32F: - case CM_SURF_FMT_DEPTH32F: - case CM_SURF_FMT_DEPTH32F_X24_STEN8: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_FLOAT; - break; - case CM_SURF_FMT_DEPTH24_STEN8: - formatType = HSA_EXT_IMAGE_CHANNEL_TYPE_UNORM_INT24; - break; - default: - assert(false); - } - - return formatType; -} - -static uint32_t GetHSAILImageOrderType(gslChannelOrder chOrder, cmSurfFmt format) { - uint32_t orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_A; - - switch (chOrder) { - case GSL_CHANNEL_ORDER_R: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_R; - break; - case GSL_CHANNEL_ORDER_A: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_A; - break; - case GSL_CHANNEL_ORDER_RG: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_RG; - break; - case GSL_CHANNEL_ORDER_RA: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_RA; - break; - case GSL_CHANNEL_ORDER_RGB: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_RGB; - break; - case GSL_CHANNEL_ORDER_RGBA: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_RGBA; - break; - case GSL_CHANNEL_ORDER_BGRA: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_BGRA; - break; - case GSL_CHANNEL_ORDER_ARGB: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_ARGB; - break; - case GSL_CHANNEL_ORDER_SRGB: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_SRGB; - break; - case GSL_CHANNEL_ORDER_SRGBX: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_SRGBX; - break; - case GSL_CHANNEL_ORDER_SRGBA: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_SRGBA; - break; - case GSL_CHANNEL_ORDER_SBGRA: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_SBGRA; - break; - case GSL_CHANNEL_ORDER_INTENSITY: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_INTENSITY; - break; - case GSL_CHANNEL_ORDER_LUMINANCE: - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_LUMINANCE; - break; - case GSL_CHANNEL_ORDER_REPLICATE_R: - if ((format == CM_SURF_FMT_DEPTH32F_X24_STEN8) || (format == CM_SURF_FMT_DEPTH24_STEN8)) { - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_DEPTH_STENCIL; - } else { - orderType = HSA_EXT_IMAGE_CHANNEL_ORDER_DEPTH; - } - break; - default: - assert(false); - } - - return orderType; -} - -bool Resource::create(MemoryType memType, CreateParams* params) { - bool calRes = false; - gslMemObject gslResource = 0; - gslMemObject gslResOriginal = 0; - const amd::HostMemoryReference* hostMemRef = NULL; - bool imageCreateView = false; - CALuint hostMemOffset = 0; - bool foundCalRef = false; - bool viewDefined = false; - uint viewLayer = 0; - uint viewLevel = 0; - uint viewFlags = 0; - gslResource3D viewSize = {0}; - size_t viewOffset = 0; - cmSurfFmt viewSurfFmt; - gslChannelOrder viewChannelOrder = GSL_CHANNEL_ORDER_UNSPECIFIED; - gslMemObjectAttribType viewResType; - CALresourceDesc desc; - uint64 bytePitch = (uint64)-1; - bool useRowPitch = false; - bool mipLevelPitchPad = false; - - desc.vaBase = 0; - desc.minAlignment = 0; - desc.isAllocExecute = false; - desc.isAllocSVM = false; - desc.section = GSL_SECTION_REGULAR; - if (NULL != params && NULL != params->owner_) { // make sure params not NULL - mcaddr svmPtr = reinterpret_cast(params->owner_->getSvmPtr()); - desc.vaBase = (svmPtr == 1) ? 0 : svmPtr; - // Dont cache coarse\fine grain svm resource as these may not be released - // and allocations may fail since there is limited space for coarse\fine grainbuffers - cal_.skipRsrcCache_ = (svmPtr != 0); - desc.section = (svmPtr != 0) ? GSL_SECTION_SVM : GSL_SECTION_REGULAR; - - if (params->owner_->getMemFlags() & CL_MEM_SVM_ATOMICS) { - desc.section = GSL_SECTION_SVM_ATOMICS; - } - - if (dev().settings().svmFineGrainSystem_ && - (desc.section == GSL_SECTION_SVM || desc.section == GSL_SECTION_SVM_ATOMICS)) { - cal_.isAllocSVM_ = desc.isAllocSVM = true; - } - } - - if (memType == Shader) { - if (dev().settings().svmFineGrainSystem_) { - cal_.isAllocExecute_ = desc.isAllocExecute = true; - cal_.isAllocSVM_ = desc.isAllocSVM = true; - } - // force to use remote memory for HW DEBUG or use - // local memory once we determine if FGS is supported - memType = (!dev().settings().enableHwDebug_) ? Local : RemoteUSWC; - } - - // This is a thread safe operation - const_cast(dev()).initializeHeapResources(); - - // Get the element size - elementSize_ = static_cast(memoryFormatSize(cal()->format_).size_); - cal_.type_ = memType; - if (memType == Scratch) { - // use local memory for scratch buffer unless it is using HW DEBUG - cal_.type_ = (!dev().settings().enableHwDebug_) ? Local : RemoteUSWC; - cal_.scratch_ = true; - } - - // Force remote allocation if it was requested in the settings - if (dev().settings().remoteAlloc_ && ((memoryType() == Local) || (memoryType() == Persistent))) { - if (dev().settings().apuSystem_ && dev().settings().viPlus_) { - cal_.type_ = Remote; - } else { - cal_.type_ = RemoteUSWC; - } - } - - if (dev().settings().disablePersistent_ && (memoryType() == Persistent)) { - cal_.type_ = RemoteUSWC; - } - - if (cal()->buffer_) { - // Force linear tiling for buffer alloctions - cal_.flags_ |= CAL_RESALLOC_GLOBAL_BUFFER; - } - - if (params != NULL) { - gpu_ = params->gpu_; - } - - switch (memoryType()) { - case Heap: - gslResource = dev().resGetHeap(0); - if (gslResource == 0) { - return false; - } - calRes = true; - cal_.width_ = static_cast(gslResource->getPitch()); - cal_.pitch_ = static_cast(gslResource->getPitch()); - break; - case Persistent: - if (dev().settings().linearPersistentImage_) { - // Force linear tiling for image allocations in persistent - cal_.flags_ |= CAL_RESALLOC_GLOBAL_BUFFER; - } - // Fall through ... - case RemoteUSWC: - case Remote: - case Shader: - case BusAddressable: - case ExternalPhysical: - // Fall through to process the memory allocation ... - case Local: { - if (cal()->buffer_) { - //! @todo Remove alignment. - //! GSL asserts in mem copy with an unaligned size - cal_.width_ = amd::alignUp(cal_.width_, 64); - if ((desc.section == GSL_SECTION_SVM || desc.section == GSL_SECTION_SVM_ATOMICS)) { - cal_.width_ = amd::alignUp(cal_.width_, 64 * Ki / sizeof(uint32_t)); - } - } - - desc.dimension = cal()->dimension_; - desc.size.width = cal()->width_; - desc.size.height = cal()->height_; - desc.size.depth = cal()->depth_; - desc.format = cal()->format_; - desc.channelOrder = cal()->channelOrder_; - desc.flags = cal()->flags_; - desc.mipLevels = cal()->mipLevels_; - desc.systemMemory = NULL; - - uint allocAttempt = 0; - do { - // Find a type for allocation - if (memoryType() == Persistent) { - desc.type = GSL_MOA_MEMORY_CARD_LOCKABLE; - } else if (memoryType() == Remote) { - desc.type = GSL_MOA_MEMORY_REMOTE_CACHEABLE; - } else if (memoryType() == RemoteUSWC) { - desc.type = GSL_MOA_MEMORY_AGP; - } else if (memoryType() == BusAddressable) { - desc.type = GSL_MOA_MEMORY_CARD_BUS_ADDRESSABLE; - } else if (memoryType() == ExternalPhysical) { - desc.type = GSL_MOA_MEMORY_CARD_EXTERNAL_PHYSICAL; - cl_bus_address_amd bus_address = - (reinterpret_cast(params->owner_))->busAddress(); - desc.busAddress[0] = bus_address.surface_bus_address; - desc.busAddress[1] = bus_address.marker_bus_address; - } else { - desc.type = GSL_MOA_MEMORY_CARD_EXT_NONEXT; - } - - // Check resource cache first for an appropriate resource - gslRef_ = dev().resourceCache().findCalResource(&cal_); - if (memType == Scratch) { - desc.minAlignment = 64 * Ki; - } else if ((gslRef_ != NULL) && (!dev().settings().use64BitPtr_)) { - // Make sure runtime didn't pick a resource with > 4GB address - if ((cal()->dimension_ == GSL_MOA_BUFFER) && - (static_cast(gslRef_->gslResource()->getSurfaceAddress() + - gslRef_->gslResource()->getSurfaceSize()) > - (uint64_t(4) * Gi))) { - gslRef_->release(); - gslRef_ = NULL; - } - } - // Try to allocate memory if we couldn't find a cached resource - if (gslRef_ == NULL) { - // Allocate memory - gslResource = dev().resAlloc(&desc); - if (gslResource != 0) { - calRes = true; - } - } else { - calRes = true; - gslResource = gslRef_->gslOriginal(); - foundCalRef = true; - } - - // If GSL fails allocation then try other heaps - if (!calRes) { - // Free cache if we failed allocation - if (dev().resourceCache().free()) { - // We freed something - attempt to allocate memory again - continue; - } - - // Local to Persistent - if (memoryType() == Local) { - cal_.type_ = Persistent; - } - // Don't switch to USWC if persistent memory was explicitly asked - else if ((allocAttempt > 0) && (memoryType() == Persistent)) { - cal_.type_ = RemoteUSWC; - } - // Remote cacheable to uncacheable - else if (memoryType() == Remote) { - cal_.type_ = RemoteUSWC; - } else { - break; - } - allocAttempt++; - } - } while (!calRes); - } break; - case Pinned: { - PinnedParams* pinned = reinterpret_cast(params); - CALuint allocSize = static_cast(pinned->size_); - void* pinAddress; - hostMemRef = pinned->hostMemRef_; - pinAddress = address_ = hostMemRef->hostMem(); - - // Use untiled allocation - cal_.flags_ |= CAL_RESALLOC_GLOBAL_BUFFER; - - desc.size.width = cal()->width_; - - if (cal()->dimension_ == GSL_MOA_BUFFER) { - // Allign offset to 4K boundary (Vista/Win7 limitation) - char* tmpHost = const_cast( - amd::alignDown(reinterpret_cast(address_), PinnedMemoryAlignment)); - - // Find the partial size for unaligned copy - hostMemOffset = static_cast(reinterpret_cast(address_) - tmpHost); - - pinOffset_ = hostMemOffset & 0xff; - - pinAddress = tmpHost; - // Align width to avoid GSL useless assert with a view - if (hostMemOffset != 0) { - desc.size.width += hostMemOffset / elementSize(); - desc.size.width = amd::alignUp(desc.size.width, 64); - } - hostMemOffset &= ~(0xff); - } else if (cal()->dimension_ == GSL_MOA_TEXTURE_2D) { - //! @todo: Width has to be aligned for 3D. - //! Need to be replaced with a compute copy - // Width aligned by 8 texels - if (((cal()->width_ % 0x8) != 0) || - // Pitch aligned by 64 bytes - (((cal()->width_ * elementSize()) % 0x40) != 0)) { - return false; - } - } else { - //! @todo GSL doesn't support pinning with resAlloc_ - return false; - } - - // Fill the GSL desc info structure - desc.dimension = cal()->dimension_; - desc.type = GSL_MOA_MEMORY_SYSTEM; - desc.size.height = cal()->height_; - desc.size.depth = cal()->depth_; - desc.format = cal()->format_; - desc.channelOrder = cal()->channelOrder_; - desc.mipLevels = 0; - desc.systemMemory = reinterpret_cast(pinAddress); - desc.flags = 0; - - // Ensure page alignment - if ((CALuint64)desc.systemMemory & (amd::Os::pageSize() - 1)) { - return false; - } - - gslResource = dev().resAlloc(&desc); - if (gslResource != 0) { - calRes = true; - } else { - pinOffset_ = 0; - } - } break; - case View: { - // Save the offset in the global heap - ViewParams* view = reinterpret_cast(params); - offset_ = view->offset_; - - // Make sure parent was provided - if (NULL != view->resource_) { - viewOwner_ = view->resource_; - uint64 bytePitch = (view->size_ + viewOwner_->pinOffset()); - viewSize.width = bytePitch / elementSize(); - viewSize.height = 1; - viewSize.depth = 1; - viewOffset = static_cast(offset() / elementSize()); - - gslResource = dev().resAllocView(view->resource_->gslResource(), viewSize, viewOffset, - cal()->format_, GSL_CHANNEL_ORDER_REPLICATE_R, - cal()->dimension_, 0, 0, cal()->flags_, bytePitch); - if (gslResource != 0) { - calRes = true; - } - - if (viewOwner_->isMemoryType(Pinned)) { - address_ = viewOwner_->data() + offset(); - } - pinOffset_ = viewOwner_->pinOffset(); - } else { - cal_.type_ = Empty; - } - } break; - case ImageView: { - ImageViewParams* imageView = reinterpret_cast(params); - imageCreateView = true; - viewLayer = imageView->layer_; - viewLevel = imageView->level_; - gslResource = imageView->resource_->gslResource(); - viewOwner_ = imageView->resource_; - if ((viewLevel != 0) || viewOwner_->mipMapped()) { - viewFlags |= CAL_RESALLOCSLICEVIEW_LEVEL; - } - if ((viewOwner_->viewOwner_ != NULL) && viewOwner_->viewOwner_->mipMapped()) { - mipLevelPitchPad = true; - } - - if (viewLayer != 0) { - viewFlags |= CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER; - } - calRes = true; - } break; - case ImageBuffer: { - ImageBufferParams* imageBuffer = reinterpret_cast(params); - imageCreateView = true; - gslResource = imageBuffer->resource_->gslResource(); - viewOwner_ = imageBuffer->resource_; - calRes = true; - useRowPitch = true; - } break; - case OGLInterop: { - OGLInteropParams* oglRes = reinterpret_cast(params); - assert(oglRes->glPlatformContext_ && "We don't have OGL context!"); - switch (oglRes->type_) { - case InteropVertexBuffer: - glType_ = GL_RESOURCE_ATTACH_VERTEXBUFFER_AMD; - break; - case InteropRenderBuffer: - glType_ = GL_RESOURCE_ATTACH_RENDERBUFFER_AMD; - break; - case InteropTexture: - case InteropTextureViewLevel: - case InteropTextureViewCube: - glType_ = GL_RESOURCE_ATTACH_TEXTURE_AMD; - break; - default: - LogError("Unknown OGL interop type!"); - return false; - break; - } - glPlatformContext_ = oglRes->glPlatformContext_; - glDeviceContext_ = oglRes->glDeviceContext_; - CALGSLDevice::GLResAssociate resData = {0}; - resData.GLContext = oglRes->glPlatformContext_; - resData.GLdeviceContext = oglRes->glDeviceContext_; - resData.name = oglRes->handle_; - resData.type = glType_; - // We need not pass any flags down to OGL for interop and there is no need to - // pass down resData.flags field - - if (dev().resGLAssociate(resData)) { - gslResource = resData.memObject; - glInteropMbRes_ = resData.mbResHandle; - glInterop_ = resData.mem_base; - calRes = true; - } - - // Check if we have to create a view - if (calRes && ((oglRes->type_ == InteropTextureViewLevel) || - (oglRes->type_ == InteropTextureViewCube))) { - imageCreateView = true; - viewLayer = oglRes->layer_; - viewLevel = oglRes->mipLevel_; - - // Find the view parameters - if (InteropTextureViewLevel == oglRes->type_) { - viewFlags |= CAL_RESALLOCSLICEVIEW_LEVEL; - } else if (InteropTextureViewCube == oglRes->type_) { - viewFlags |= CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER; - } else { - LogError("Unknown Interop View Type"); - } - } - } break; -#ifdef _WIN32 - case D3D9Interop: - case D3D10Interop: - case D3D11Interop: { - D3DInteropParams* d3dRes = reinterpret_cast(params); - desc.dimension = cal()->dimension_; - desc.size.width = cal()->width_; - desc.size.height = cal()->height_; - desc.size.depth = cal()->depth_; - desc.format = cal()->format_; - desc.channelOrder = cal()->channelOrder_; - desc.flags = cal()->flags_; - desc.mipLevels = 0; - desc.systemMemory = NULL; - switch (d3dRes->misc) { - case 1: // NV12 format - case 2: // YV12 format - // Readjust the size to the original NV12/YV12 size, since runtime - // creates an interop for all planes - switch (d3dRes->layer_) { - case 0: - desc.size.height = 3 * desc.size.height / 2; - break; - case 1: - case 2: - // Force R8 format for the interop allocation by default - if (1 == d3dRes->misc) { - desc.format = CM_SURF_FMT_R8; - desc.channelOrder = GSL_CHANNEL_ORDER_R; - } - desc.size.width = 2 * desc.size.width; - desc.size.height = 3 * desc.size.height; - break; - default: - break; - } - break; - default: - break; - } - - // Create an interop GSL object - gslResource = - dev().resMapD3DResource(&desc, (CALuint64)d3dRes->handle_, (memoryType() != D3D9Interop)); - if (gslResource != 0) { - calRes = true; - } else { - return false; - } - - - // Check if we have to create a view - if (calRes && ((d3dRes->type_ == InteropTextureViewLevel) || - (d3dRes->type_ == InteropTextureViewCube))) { - imageCreateView = true; - viewLayer = d3dRes->layer_; - viewLevel = d3dRes->mipLevel_; - - // Find the view parameters - if (InteropTextureViewLevel == d3dRes->type_) { - viewFlags |= CAL_RESALLOCSLICEVIEW_LEVEL; - } else if (InteropTextureViewCube == d3dRes->type_) { - viewFlags |= CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER; - } else { - LogError("Unknown Interop View Type"); - } - } - - switch (d3dRes->misc) { - case 0: - break; - case 1: // NV12 format - case 2: // YV12 format - // Create a view for the specified plane - viewDefined = true; - viewSize.width = cal()->width_; - viewSize.height = cal()->height_; - viewSize.depth = 1; - bytePitch = static_cast(gslResource->getPitch()); - viewOffset = 0; - viewSurfFmt = cal()->format_; - viewChannelOrder = cal()->channelOrder_; - switch (d3dRes->layer_) { - case -1: - bytePitch *= elementSize(); - break; - case 0: - bytePitch *= elementSize(); - break; - case 1: - // Y - plane size to the offset - viewOffset = bytePitch * viewSize.height * 2; - if (d3dRes->misc == 2) { - // YV12 format U is 2 times smaller plane - bytePitch /= 2; - } - break; - case 2: - // Y + U plane sizes to the offest. - // U plane is 4 times smaller than Y => 5/2 - viewOffset = bytePitch * viewSize.height * 5 / 2; - // V is 2 times smaller plane - bytePitch /= 2; - break; - default: - LogError("Unknown Interop View Type"); - calRes = false; - break; - } - break; - case 3: - break; - default: - LogError("Unknown Interop View Type"); - calRes = false; - } - } break; -#endif // _WIN32 - default: - LogWarning("Resource::create() called with unknown memory type"); - return false; - break; - } - - // Create a view for interop, since the original buffer may have different format - // than the global buffer and GSL mem copy will fail - bool interopBufView = - cal()->buffer_ && ((memoryType() == D3D10Interop) || (memoryType() == OGLInterop) || - (memoryType() == D3D11Interop)); - - bool ignoreParentHandle = ((memoryType() == ImageView) || (memoryType() == ImageBuffer)); - - // Create imageview if it was requested - if (calRes && (imageCreateView || interopBufView || hostMemOffset || viewDefined)) { - gslResOriginal = gslResource; - - // Disable tiling if it's a buffer view - if (interopBufView || hostMemOffset) { - viewFlags = CAL_RESALLOCVIEW_GLOBAL_BUFFER; - } - - viewResType = cal()->dimension_; - if (!viewDefined) { - viewSize.width = cal()->width_ + (pinOffset() / elementSize()); - viewSize.height = cal()->height_; - viewSize.depth = cal()->depth_; - viewOffset = hostMemOffset / static_cast(elementSize()); - viewSurfFmt = cal()->format_; - viewChannelOrder = cal()->channelOrder_; - } - - if (useRowPitch && (params->owner_ != NULL) && params->owner_->asImage() && - (params->owner_->asImage()->getRowPitch() != 0)) { - bytePitch = params->owner_->asImage()->getRowPitch(); - } - - // Allocate a view resource object - gslResource = - dev().resAllocView(gslResOriginal, viewSize, viewOffset, viewSurfFmt, viewChannelOrder, - viewResType, viewLevel, viewLayer, viewFlags, bytePitch); - - if (gslResource == 0) { - // If we don't have to keep the parent handle, - // then destroy the original resource - if (!ignoreParentHandle) { - dev().resFree(gslResOriginal); - gslResOriginal = 0; - } - LogError("ResAlloc failed!"); - return false; - } - - if (ignoreParentHandle) { - gslResOriginal = 0; - } - } - - if (!calRes) { - if (gslResource != 0) { - dev().resFree(gslResource); - } - if (memoryType() != Pinned) { - LogError("calResAlloc failed!"); - } - return false; - } - - // Find memory location - switch (gslResource->getAttribs().location) { - case GSL_MOA_MEMORY_CARD: - case GSL_MOA_MEMORY_CARD_EXT: - case GSL_MOA_MEMORY_CARD_LOCKABLE: - case GSL_MOA_MEMORY_CARD_EXT_NONEXT: - case GSL_MOA_MEMORY_CARD_BUS_ADDRESSABLE: - cal_.cardMemory_ = true; - break; - default: - cal_.cardMemory_ = false; - break; - } - - gslMemObjectAttribTiling tiling = gslResource->getAttribs().tiling; - cal_.tiled_ = (GSL_MOA_TILING_LINEAR != tiling) && (GSL_MOA_TILING_LINEAR_GENERAL != tiling); - - // Get the heap block offset - hbOffset_ = gslResource->getSurfaceAddress() - dev().heap().baseAddress(); - hbSize_ = static_cast(gslResource->getSurfaceSize()); - - if (!dev().settings().use64BitPtr_ && - !((memType == Scratch) || ((memType == View) && viewOwner_->cal()->scratch_))) { - // Make sure runtime doesn't go over the address space limit for buffers - if ((memoryType() != Heap) && (cal()->dimension_ == GSL_MOA_BUFFER) && - ((hbOffset_ + hbSize_) > (uint64_t(4) * Gi))) { - if (cal_.cardMemory_) { - LogPrintfError("Out of 4GB address space. Base: 0x%016llX, size: 0x%016llX!", hbOffset_, - hbSize_); - - dev().resFree(gslResource); - //! @note: A workaround for a Windows delay on memory destruction - //! Runtime submits a fake memory fill to force KMD to return - //! the freed memory ranges - if (IS_WINDOWS) { - uint32_t pattern = 0; - Memory* dummy = reinterpret_cast(dev().dummyPage()->getDeviceMemory(dev())); - dev().xferMgr().fillBuffer(*dummy, &pattern, sizeof(uint32_t), amd::Coord3D(0), - amd::Coord3D(sizeof(uint32_t))); - } - if ((gslResOriginal != 0) && !ignoreParentHandle) { - dev().resFree(gslResOriginal); - gslResOriginal = 0; - } - return false; - } else { - LogWarning("Out of 4GB address space for AHP/UHP!"); - } - } - } - - if (!foundCalRef) { - gslRef_ = new GslResourceReference(dev(), gslResource, gslResOriginal); - if (gslRef_ == NULL) { - LogError("Memory allocation failure!"); - dev().resFree(gslResource); - return false; - } - } - - if (!cal()->buffer_) { - hwSrd_ = dev().srds().allocSrdSlot(reinterpret_cast(&hwState_)); - if (0 == hwSrd_) { - return false; - } - dev().fillImageHwState(gslResource, hwState_, 8 * sizeof(uint32_t)); - hwState_[8] = GetHSAILImageFormatType(cal()->format_); - hwState_[9] = GetHSAILImageOrderType(cal()->channelOrder_, cal()->format_); - hwState_[10] = static_cast(cal()->width_); - if (memoryType() == ImageView) { - // Workaround for depth view, change tileIndex to the parent for depth view - if (viewChannelOrder == GSL_CHANNEL_ORDER_REPLICATE_R) { - if ((hwState_[3] & 0x1f00000) == 0xe00000) { - hwState_[3] = (hwState_[3] & 0xfe0fffff) | (viewOwner_->hwState_[3] & 0x1f00000); - } - } - // Update the POW2_PAD flag, otherwise HW uses a wrong pitch value - if ((viewFlags & CAL_RESALLOCSLICEVIEW_LEVEL) || mipLevelPitchPad) { - hwState_[3] |= (viewOwner_->hwState_[3] & 0x2000000); - } - } - hwState_[11] = 0; // one extra reserved field in the argument - } - - if (desc.section == GSL_SECTION_SVM || desc.section == GSL_SECTION_SVM_ATOMICS) { - params->owner_->setSvmPtr(reinterpret_cast(gslResource->getSurfaceAddress())); - } - - return true; -} - -void Resource::free() { - if (gslRef_ == NULL) { - return; - } - - // Sanity check for the map calls - if (mapCount_ != 0) { - LogWarning("Resource wasn't unlocked, but destroyed!"); - } - const bool wait = (memoryType() != ImageView) && (memoryType() != ImageBuffer); - - // Check if resource could be used in any queue(thread) - if (gpu_ == NULL) { - Device::ScopedLockVgpus lock(dev()); - - if (renames_.size() == 0) { - // Destroy GSL resource - if (gslResource() != 0) { - // Release all virtual memory objects on all virtual GPUs - for (uint idx = 0; idx < dev().vgpus().size(); ++idx) { - // Ignore the transfer queue, - // since it releases resources after every operation - if (dev().vgpus()[idx] != dev().xferQueue()) { - dev().vgpus()[idx]->releaseMemory(gslResource(), wait); - } - } - - //! @note: This is a workaround for bad applications that - //! don't unmap memory - if (mapCount_ != 0) { - unmap(NULL); - } - - // Add resource to the cache - if (!dev().resourceCache().addCalResource(&cal_, gslRef_)) { - gslFree(); - } - } - } else { - renames_[curRename_]->cpuAddress_ = 0; - for (size_t i = 0; i < renames_.size(); ++i) { - gslRef_ = renames_[i]; - // Destroy GSL resource - if (gslResource() != 0) { - // Release all virtual memory objects on all virtual GPUs - for (uint idx = 0; idx < dev().vgpus().size(); ++idx) { - // Ignore the transfer queue, - // since it releases resources after every operation - if (dev().vgpus()[idx] != dev().xferQueue()) { - dev().vgpus()[idx]->releaseMemory(gslResource()); - } - } - gslFree(); - } - } - } - } else { - if (renames_.size() == 0) { - // Destroy GSL resource - if (gslResource() != 0) { - // Release virtual memory object on the specified virtual GPU - gpu_->releaseMemory(gslResource(), wait); - gslFree(); - } - } else - for (size_t i = 0; i < renames_.size(); ++i) { - gslRef_ = renames_[i]; - // Destroy GSL resource - if (gslResource() != 0) { - // Release virtual memory object on the specified virtual GPUs - gpu_->releaseMemory(gslResource()); - gslFree(); - } - } - } - - // Free SRD for images - if (!cal()->buffer_) { - dev().srds().freeSrdSlot(hwSrd_); - } -} - -void Resource::writeRawData(VirtualGPU& gpu, size_t size, const void* data, - bool waitForEvent) const { - GpuEvent event; - - // Write data size bytes to surface - // size needs to be DWORD aligned - assert((size & 3) == 0); - gpu.eventBegin(MainEngine); - gslResource()->writeDataRaw(gpu.cs(), size, data, true); - gpu.eventEnd(MainEngine, event); - - setBusy(gpu, event); - // Update the global GPU event - gpu.setGpuEvent(event, false); - - if (waitForEvent) { - // Wait for event to complete - gpu.waitForEvent(&event); - } -} - -bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin, - const amd::Coord3D& dstOrigin, const amd::Coord3D& size, - Resource& dstResource, bool enableCopyRect, bool flushDMA, - uint bytesPerElement) const { - GpuEvent event; - bool result; - CALuint syncFlags = CAL_MEMCOPY_SYNC; - EngineType activeEngineID = gpu.engineID_; - static const bool waitOnBusyEngine = true; - // \note timing issues in Linux with sync mode - bool flush = true; - - // Check if runtime can use async memory copy, - // even if a caller didn't request async - // Keep ASYNC if profiling is disabled or sdma profiling is possible - if ((!gpu.profiling() || dev().settings().sdmaProfiling_) && - (!cal()->cardMemory_ || !dstResource.cal()->cardMemory_)) { - // Switch to SDMA engine - gpu.engineID_ = SdmaEngine; - syncFlags = CAL_MEMCOPY_ASYNC; - flush = false; - } - - // Wait for the resources, since runtime may use async transfers - wait(gpu, waitOnBusyEngine); - dstResource.wait(gpu, waitOnBusyEngine); - - size_t calSrcOrigin[3], calDstOrigin[3], calSize[3]; - calSrcOrigin[0] = srcOrigin[0] + pinOffset(); - calSrcOrigin[1] = srcOrigin[1]; - calSrcOrigin[2] = srcOrigin[2]; - calDstOrigin[0] = dstOrigin[0] + dstResource.pinOffset(); - calDstOrigin[1] = dstOrigin[1]; - calDstOrigin[2] = dstOrigin[2]; - calSize[0] = size[0]; - calSize[1] = size[1]; - calSize[2] = size[2]; - - result = gpu.copyPartial(event, gslResource(), calSrcOrigin, dstResource.gslResource(), - calDstOrigin, calSize, static_cast(syncFlags), - enableCopyRect, bytesPerElement); - - if (result) { - // Mark source and destination as busy - setBusy(gpu, event); - dstResource.setBusy(gpu, event); - - // Update the global GPU event - gpu.setGpuEvent(event, (flush | flushDMA)); - } - - // Restore the original engine - gpu.engineID_ = activeEngineID; - - return result; -} - -void Resource::setBusy(VirtualGPU& gpu, GpuEvent gpuEvent) const { - gpu.assignGpuEvent(gslResource(), gpuEvent); - - // If current resource is a view, then update the parent event as well - if (viewOwner_ != NULL) { - viewOwner_->setBusy(gpu, gpuEvent); - } -} - -void Resource::wait(VirtualGPU& gpu, bool waitOnBusyEngine) const { - GpuEvent* gpuEvent = gpu.getGpuEvent(gslResource()); - - // Check if we have to wait unconditionally - if (!waitOnBusyEngine || - // or we have to wait only if another engine was used on this resource - (gpuEvent->engineId_ != gpu.engineID_)) { - gpu.waitForEvent(gpuEvent); - } - - // If current resource is a view and not in the global heap, - // then wait for the parent event as well - if ((viewOwner_ != NULL) && (viewOwner_ != &dev().globalMem())) { - viewOwner_->wait(gpu, waitOnBusyEngine); - } -} - -bool Resource::hostWrite(VirtualGPU* gpu, const void* hostPtr, const amd::Coord3D& origin, - const amd::Coord3D& size, uint flags, size_t rowPitch, size_t slicePitch) { - void* dst; - - size_t startLayer = origin[2]; - size_t numLayers = size[2]; - if (cal()->dimension_ == GSL_MOA_TEXTURE_1D_ARRAY) { - startLayer = origin[1]; - numLayers = size[1]; - } - - // Get physical GPU memmory - dst = map(gpu, flags, startLayer, numLayers); - if (NULL == dst) { - LogError("Couldn't map GPU memory for host write"); - return false; - } - - if (1 == cal()->dimSize_) { - size_t copySize = (cal()->buffer_) ? size[0] : size[0] * elementSize_; - - // Update the pointer - dst = static_cast(static_cast(dst) + origin[0]); - - // Copy memory - amd::Os::fastMemcpy(dst, hostPtr, copySize); - } else { - size_t srcOffs = 0; - size_t dstOffsBase = origin[0] * elementSize_; - size_t dstOffs; - - // Make sure we use the right pitch if it's not specified - if (rowPitch == 0) { - rowPitch = size[0] * elementSize_; - } - - // Make sure we use the right slice if it's not specified - if (slicePitch == 0) { - slicePitch = size[0] * size[1] * elementSize_; - } - - // Adjust the destination offset with Y dimension - dstOffsBase += cal()->pitch_ * origin[1] * elementSize_; - - // Adjust the destination offset with Z dimension - dstOffsBase += cal()->slice_ * origin[2] * elementSize_; - - // Copy memory slice by slice - for (size_t slice = 0; slice < size[2]; ++slice) { - dstOffs = dstOffsBase + slice * cal()->slice_ * elementSize_; - srcOffs = slice * slicePitch; - - // Copy memory line by line - for (size_t row = 0; row < size[1]; ++row) { - // Copy memory - amd::Os::fastMemcpy((reinterpret_cast
(dst) + dstOffs), - (reinterpret_cast(hostPtr) + srcOffs), - size[0] * elementSize_); - - dstOffs += cal()->pitch_ * elementSize_; - srcOffs += rowPitch; - } - } - } - - // Unmap GPU memory - unmap(gpu); - - return true; -} - -bool Resource::hostRead(VirtualGPU* gpu, void* hostPtr, const amd::Coord3D& origin, - const amd::Coord3D& size, size_t rowPitch, size_t slicePitch) { - void* src; - - size_t startLayer = origin[2]; - size_t numLayers = size[2]; - if (cal()->dimension_ == GSL_MOA_TEXTURE_1D_ARRAY) { - startLayer = origin[1]; - numLayers = size[1]; - } - - // Get physical GPU memmory - src = map(gpu, ReadOnly, startLayer, numLayers); - if (NULL == src) { - LogError("Couldn't map GPU memory for host read"); - return false; - } - - if (1 == cal()->dimSize_) { - size_t copySize = (cal()->buffer_) ? size[0] : size[0] * elementSize_; - - // Update the pointer - src = static_cast(static_cast(src) + origin[0]); - - // Copy memory - amd::Os::fastMemcpy(hostPtr, src, copySize); - } else { - size_t srcOffsBase = origin[0] * elementSize_; - size_t srcOffs; - size_t dstOffs = 0; - - // Make sure we use the right pitch if it's not specified - if (rowPitch == 0) { - rowPitch = size[0] * elementSize_; - } - - // Make sure we use the right slice if it's not specified - if (slicePitch == 0) { - slicePitch = size[0] * size[1] * elementSize_; - } - - // Adjust destination offset with Y dimension - srcOffsBase += cal()->pitch_ * origin[1] * elementSize_; - - // Adjust the destination offset with Z dimension - srcOffsBase += cal()->slice_ * origin[2] * elementSize_; - - // Copy memory line by line - for (size_t slice = 0; slice < size[2]; ++slice) { - srcOffs = srcOffsBase + slice * cal()->slice_ * elementSize_; - dstOffs = slice * slicePitch; - - // Copy memory line by line - for (size_t row = 0; row < size[1]; ++row) { - // Copy memory - amd::Os::fastMemcpy((reinterpret_cast
(hostPtr) + dstOffs), - (reinterpret_cast(src) + srcOffs), - size[0] * elementSize_); - - srcOffs += cal()->pitch_ * elementSize_; - dstOffs += rowPitch; - } - } - } - - // Unmap GPU memory - unmap(gpu); - - return true; -} - -void* Resource::gslMap(size_t* pitch, gslMapAccessType flags, gslMemObject resource) const { - if (cal_.cardMemory_ || cal_.tiled_) { - // @todo remove const cast - return const_cast(dev()).resMapLocal(*pitch, resource, flags); - } else { - return dev().resMapRemote(*pitch, resource, flags); - } -} - -void Resource::gslUnmap(gslMemObject resource) const { - if (cal_.cardMemory_) { - // @todo remove const cast - const_cast(dev()).resUnmapLocal(resource); - } else { - dev().resUnmapRemote(resource); - } -} - -bool Resource::gslGLAcquire() { - bool retVal = true; - if (cal()->type_ == OGLInterop) { - retVal = dev().resGLAcquire(glPlatformContext_, glInteropMbRes_, glType_); - } - return retVal; -} - -bool Resource::gslGLRelease() { - bool retVal = true; - if (cal()->type_ == OGLInterop) { - retVal = dev().resGLRelease(glPlatformContext_, glInteropMbRes_, glType_); - } - return retVal; -} -void Resource::gslFree() const { - if (cal()->type_ == OGLInterop) { - if (0 == gslRef_->resOriginal_) { - dev().resGLFree(glPlatformContext_, glDeviceContext_, gslRef_->resource_, glInterop_, - glInteropMbRes_, glType_); - gslRef_->resource_ = 0; - } else { - dev().resFree(gslRef_->resource_); - gslRef_->resource_ = 0; - dev().resGLFree(glPlatformContext_, glDeviceContext_, gslRef_->resOriginal_, glInterop_, - glInteropMbRes_, glType_); - gslRef_->resOriginal_ = 0; - } - } - gslRef_->release(); -} - -bool Resource::isMemoryType(MemoryType memType) const { - if (memoryType() == memType) { - return true; - } else if (memoryType() == View) { - return viewOwner_->isMemoryType(memType); - } - - return false; -} - -bool Resource::isPersistentDirectMap() const { - bool directMap = - ((memoryType() == Resource::Persistent) && (cal()->dimSize_ < 3) && !cal()->imageArray_); - - // If direct map is possible, then validate it with the current tiling - if (directMap && cal()->tiled_) { - //!@note IOL for Linux doesn't support tiling aperture - // and runtime doesn't force linear images in persistent - directMap = IS_WINDOWS && !dev().settings().linearPersistentImage_; - } - - return directMap; -} - -void* Resource::map(VirtualGPU* gpu, uint flags, uint startLayer, uint numLayers) { - if (isMemoryType(Pinned)) { - // Check if we have to wait - if (!(flags & NoWait)) { - if (gpu != NULL) { - wait(*gpu); - } - } - return address_; - } - - gslMapAccessType mapFlags = GSL_MAP_READ_WRITE; - - if (flags & ReadOnly) { - assert(!(flags & Discard) && "We can't use lock discard with read only!"); - mapFlags = GSL_MAP_READ_ONLY; - } - - if (flags & WriteOnly) { - mapFlags = GSL_MAP_WRITE_ONLY; - } - - // Check if use map discard - if (flags & Discard) { - mapFlags = GSL_MAP_WRITE_ONLY; - if (gpu != NULL) { - // If we use a new renamed allocation, then skip the wait - if (rename(*gpu)) { - flags |= NoWait; - } - } - } - - // Check if we have to wait - if (!(flags & NoWait)) { - if (gpu != NULL) { - wait(*gpu); - } - } - - // Check if memory wasn't mapped yet - if (++mapCount_ == 1) { - if ((cal()->dimSize_ == 3) || cal()->imageArray_ || - ((cal()->type_ == ImageView) && viewOwner_->mipMapped())) { - // Save map info for multilayer map/unmap - startLayer_ = startLayer; - numLayers_ = numLayers; - mapFlags_ = mapFlags; - // Map with layers - address_ = mapLayers(gpu, mapFlags); - } else { - // Map current resource - address_ = gslMap(&cal_.pitch_, mapFlags, gslResource()); - if (address_ == NULL) { - LogError("cal::ResMap failed!"); - --mapCount_; - return NULL; - } - } - } - - //! \note the atomic operation with counter doesn't - // guarantee that the address will be valid, - // since GSL could still process the first map - if (address_ == NULL) { - for (uint i = 0; address_ == NULL && i < 10; ++i) { - amd::Os::sleep(1); - } - assert((address_ != NULL) && "Multiple maps failed!"); - } - - return address_; -} - -void* Resource::mapLayers(VirtualGPU* gpu, CALuint flags) { - size_t srcOffs = 0; - size_t dstOffs = 0; - gslMemObject sliceResource = 0; - gslMemObjectAttribType gslDim = GSL_MOA_TEXTURE_2D; - size_t layers = cal()->depth_; - size_t height = cal()->height_; - - // Use 1D layers - if (GSL_MOA_TEXTURE_1D_ARRAY == cal()->dimension_) { - gslDim = GSL_MOA_TEXTURE_1D; - height = 1; - layers = cal()->height_; - } - - cal_.pitch_ = cal()->width_; - cal_.slice_ = cal()->pitch_ * height; - address_ = new char[cal()->slice_ * layers * elementSize()]; - if (NULL == address_) { - return NULL; - } - - // Check if map is write only - if (flags == GSL_MAP_WRITE_ONLY) { - return address_; - } - - if (numLayers_ != 0) { - layers = startLayer_ + numLayers_; - } - - dstOffs = startLayer_ * cal()->slice_ * elementSize(); - - // Loop through all layers - for (uint i = startLayer_; i < layers; ++i) { - gslResource3D gslSize; - size_t calOffset; - void* sliceAddr; - size_t pitch; - - // Allocate a layer from the image - gslSize.width = cal()->width_; - gslSize.height = height; - gslSize.depth = 1; - calOffset = 0; - sliceResource = - dev().resAllocView(gslResource(), gslSize, calOffset, cal()->format_, cal()->channelOrder_, - gslDim, 0, i, CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER); - if (0 == sliceResource) { - LogError("Map layer. resAllocSliceView failed!"); - return NULL; - } - - // Map 2D layer - sliceAddr = gslMap(&pitch, GSL_MAP_READ_ONLY, sliceResource); - if (sliceAddr == NULL) { - LogError("Map layer. CalResMap failed!"); - return NULL; - } - - srcOffs = 0; - // Copy memory line by line - for (size_t rows = 0; rows < height; ++rows) { - // Copy memory - amd::Os::fastMemcpy((reinterpret_cast
(address_) + dstOffs), - (reinterpret_cast(sliceAddr) + srcOffs), - cal()->width_ * elementSize_); - - dstOffs += cal()->pitch_ * elementSize(); - srcOffs += pitch * elementSize(); - } - - // Unmap a layer - gslUnmap(sliceResource); - dev().resFree(sliceResource); - } - - return address_; -} - -void Resource::unmap(VirtualGPU* gpu) { - if (isMemoryType(Pinned)) { - return; - } - - // Decrement map counter - int count = --mapCount_; - - // Check if it's the last unmap - if (count == 0) { - if ((cal()->dimSize_ == 3) || cal()->imageArray_ || - ((cal()->type_ == ImageView) && viewOwner_->mipMapped())) { - // Unmap layers - unmapLayers(gpu); - } else { - // Unmap current resource - gslUnmap(gslResource()); - } - address_ = NULL; - } else if (count < 0) { - LogError("dev().serialCalResUnmap failed!"); - ++mapCount_; - return; - } -} - -void Resource::unmapLayers(VirtualGPU* gpu) { - size_t srcOffs = 0; - size_t dstOffs = 0; - gslMemObjectAttribType gslDim = GSL_MOA_TEXTURE_2D; - gslMemObject sliceResource = NULL; - CALuint layers = cal()->depth_; - CALuint height = cal()->height_; - - // Use 1D layers - if (GSL_MOA_TEXTURE_1D_ARRAY == cal()->dimension_) { - gslDim = GSL_MOA_TEXTURE_1D; - height = 1; - layers = cal()->height_; - } - - if (numLayers_ != 0) { - layers = startLayer_ + numLayers_; - } - - srcOffs = startLayer_ * cal()->slice_ * elementSize(); - - // Check if map is write only - if (!(mapFlags_ == GSL_MAP_READ_ONLY)) { - // Loop through all layers - for (uint i = startLayer_; i < layers; ++i) { - gslResource3D gslSize; - size_t calOffset; - void* sliceAddr; - size_t pitch; - - // Allocate a layer from the image - gslSize.width = cal()->width_; - gslSize.height = height; - gslSize.depth = 1; - calOffset = 0; - sliceResource = dev().resAllocView(gslResource(), gslSize, calOffset, cal()->format_, - cal()->channelOrder_, gslDim, 0, i, - CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER); - if (0 == sliceResource) { - LogError("Unmap layer. resAllocSliceView failed!"); - return; - } - - // Map a layer - sliceAddr = gslMap(&pitch, GSL_MAP_WRITE_ONLY, sliceResource); - if (sliceAddr == NULL) { - LogError("Unmap layer. CalResMap failed!"); - return; - } - - dstOffs = 0; - // Copy memory line by line - for (size_t rows = 0; rows < height; ++rows) { - // Copy memory - amd::Os::fastMemcpy((reinterpret_cast
(sliceAddr) + dstOffs), - (reinterpret_cast(address_) + srcOffs), - cal()->width_ * elementSize_); - - dstOffs += pitch * elementSize(); - srcOffs += cal()->pitch_ * elementSize(); - } - - // Unmap a layer - gslUnmap(sliceResource); - dev().resFree(sliceResource); - } - } - - // Destroy the mapped memory - delete[] reinterpret_cast(address_); -} - -void Resource::setActiveRename(VirtualGPU& gpu, GslResourceReference* rename) { - // Copy the unique GSL data - gslRef_ = rename; - address_ = rename->cpuAddress_; - - hbOffset_ = rename->gslResource()->getSurfaceAddress() - dev().heap().baseAddress(); -} - -bool Resource::getActiveRename(VirtualGPU& gpu, GslResourceReference** rename) { - // Copy the old data to the rename descriptor - *rename = gslRef_; - return true; -} - -bool Resource::rename(VirtualGPU& gpu, bool force) { - GpuEvent* gpuEvent = gpu.getGpuEvent(gslResource()); - if (!gpuEvent->isValid() && !force) { - return true; - } - - bool useNext = false; - CALuint resSize = cal()->width_ * ((cal()->height_) ? cal()->height_ : 1) * elementSize_; - - // Rename will work with real GSL resources - if (((memoryType() != Local) && (memoryType() != Persistent) && (memoryType() != Remote) && - (memoryType() != RemoteUSWC)) || - (dev().settings().maxRenames_ == 0)) { - return false; - } - - // If the resource for renaming is too big, then lets check the current status first - // at the cost of an extra flush - if (resSize >= (dev().settings().maxRenameSize_ / dev().settings().maxRenames_)) { - if (gpu.isDone(gpuEvent)) { - return true; - } - } - - // Save the first - if (renames_.size() == 0) { - GslResourceReference* rename; - if (mapCount_ > 0) { - gslRef_->cpuAddress_ = address_; - } - if (!getActiveRename(gpu, &rename)) { - return false; - } - - curRename_ = renames_.size(); - renames_.push_back(rename); - } - - // Can we use a new rename? - if ((renames_.size() <= dev().settings().maxRenames_) && - ((renames_.size() * resSize) <= dev().settings().maxRenameSize_)) { - GslResourceReference* rename; - - // Create a new GSL allocation - if (create(memoryType())) { - if (mapCount_ > 0) { - assert(!cal()->cardMemory_ && "Unsupported memory type!"); - gslRef_->cpuAddress_ = dev().resMapRemote(cal_.pitch_, gslResource(), GSL_MAP_READ_WRITE); - if (gslRef_->cpuAddress_ == NULL) { - LogError("gslMap fails on rename!"); - } - address_ = gslRef_->cpuAddress_; - } - if (getActiveRename(gpu, &rename)) { - curRename_ = renames_.size(); - renames_.push_back(rename); - } else { - gslRef_->release(); - useNext = true; - } - } else { - useNext = true; - } - } else { - useNext = true; - } - - if (useNext) { - // Get the last submitted - curRename_++; - if (curRename_ >= renames_.size()) { - curRename_ = 0; - } - setActiveRename(gpu, renames_[curRename_]); - return false; - } - - return true; -} - -void Resource::warmUpRenames(VirtualGPU& gpu) { - for (uint i = 0; i < dev().settings().maxRenames_; ++i) { - // EPR #411675 - On Kaveri, benchmark "photo editing" of PCMarks takes longer time - // if writing 0 for the buffer paging by VidMM is excuted. Not sure how PCMarks measures it. - // Disable this code for apu - if (!dev().settings().apuSystem_) { - uint dummy = 0; - const bool NoWait = false; - // Write 0 for the buffer paging by VidMM - writeRawData(gpu, sizeof(dummy), &dummy, NoWait); - } - const bool Force = true; - rename(gpu, Force); - } -} - -ResourceCache::~ResourceCache() { free(); } - -//! \note the cache works in FILO mode -bool ResourceCache::addCalResource(Resource::CalResourceDesc* desc, GslResourceReference* ref) { - amd::ScopedLock l(&lockCacheOps_); - bool result = false; - size_t size = getResourceSize(desc); - - // Make sure current allocation isn't bigger than cache - if (((desc->type_ == Resource::Local) || (desc->type_ == Resource::Persistent) || - (desc->type_ == Resource::Remote) || (desc->type_ == Resource::RemoteUSWC)) && - (size < cacheSizeLimit_) && !desc->skipRsrcCache_) { - // Validate the cache size limit. Loop until we have enough space - while ((cacheSize_ + size) > cacheSizeLimit_) { - removeLast(); - } - Resource::CalResourceDesc* descCached = new Resource::CalResourceDesc; - if (descCached != NULL) { - // Copy the original desc to the cached version - memcpy(descCached, desc, sizeof(Resource::CalResourceDesc)); - - // Add the current resource to the cache - resCache_.push_front({descCached, ref}); - cacheSize_ += size; - if (desc->type_ == Resource::Local) { - lclCacheSize_ += size; - } - result = true; - } - } - - return result; -} - -GslResourceReference* ResourceCache::findCalResource(Resource::CalResourceDesc* desc) { - amd::ScopedLock l(&lockCacheOps_); - GslResourceReference* ref = NULL; - size_t size = getResourceSize(desc); - - // Early exit if resource is too big or it is for scratch buffer - if (size >= cacheSizeLimit_ || desc->skipRsrcCache_ || desc->scratch_) { - //! \note we may need to free the cache here to reduce memory pressure - return ref; - } - - // Serach the right resource through the cache list - for (const auto& it : resCache_) { - Resource::CalResourceDesc* entry = it.first; - // Find if we can reuse this entry - if ((entry->dimension_ == desc->dimension_) && (entry->type_ == desc->type_) && - (entry->width_ == desc->width_) && (entry->height_ == desc->height_) && - (entry->depth_ == desc->depth_) && (entry->channelOrder_ == desc->channelOrder_) && - (entry->format_ == desc->format_) && (entry->flags_ == desc->flags_) && - (entry->mipLevels_ == desc->mipLevels_) && (entry->isAllocSVM_ == desc->isAllocSVM_) && - (entry->isAllocExecute_ == desc->isAllocExecute_)) { - ref = it.second; - cacheSize_ -= size; - if (entry->type_ == Resource::Local) { - lclCacheSize_ -= size; - } - delete it.first; - // Remove the found etry from the cache - resCache_.remove(it); - break; - } - } - - return ref; -} - -bool ResourceCache::free(size_t minCacheEntries) { - amd::ScopedLock l(&lockCacheOps_); - bool result = false; - - if (minCacheEntries < resCache_.size()) { - if (static_cast(cacheSize_) > 0) { - result = true; - } - // Clear the cache - while (static_cast(cacheSize_) > 0) { - removeLast(); - } - CondLog((cacheSize_ != 0), "Incorrect size for cache release!"); - } - return result; -} - -size_t ResourceCache::getResourceSize(Resource::CalResourceDesc* desc) { - // Find the total amount of elements - size_t size = - desc->width_ * ((desc->height_) ? desc->height_ : 1) * ((desc->depth_) ? desc->depth_ : 1); - - // Find total size in bytes - size *= static_cast(memoryFormatSize(desc->format_).size_); - - return size; -} - -void ResourceCache::removeLast() { - std::pair entry; - entry = resCache_.back(); - resCache_.pop_back(); - - size_t size = getResourceSize(entry.first); - - cacheSize_ -= size; - if (entry.first->type_ == Resource::Local) { - lclCacheSize_ -= size; - } - - // Delete CalResourceDesc - delete entry.first; - - // Destroy GSL resource - entry.second->release(); -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuresource.hpp b/projects/clr/rocclr/device/gpu/gpuresource.hpp deleted file mode 100644 index 52a1811c72..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuresource.hpp +++ /dev/null @@ -1,501 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPURESOURCE_HPP_ -#define GPURESOURCE_HPP_ - -#include "platform/command.hpp" -#include "platform/program.hpp" -#include "device/gpu/gpudefs.hpp" - -#include - -//! \namespace gpu GPU Resource Implementation -namespace gpu { - -class Device; -class VirtualGPU; - -/*! \addtogroup GPU GPU Resource Implementation - * @{ - */ - -class GslResourceReference : public amd::ReferenceCountedObject { - public: - //! Default constructor - GslResourceReference(const Device& gpuDev, //!< GPU device object - gslMemObject gslResource, //!< CAL resource - gslMemObject gslResOriginal = NULL //!< Original CAL resource - ); - - //! Get CAL resource - gslMemObject gslResource() const { return resource_; } - - //! Original CAL resource - gslMemObject gslOriginal() const { return (resOriginal_ == 0) ? resource_ : resOriginal_; } - - const Device& device_; //!< GPU device - gslMemObject resource_; //!< GSL resource object - gslMemObject resOriginal_; //!< Original resource object, NULL if no channel order - void* cpuAddress_; //!< CPU address of this memory - - protected: - //! Default destructor - ~GslResourceReference(); - - private: - //! Disable copy constructor - GslResourceReference(const GslResourceReference&); - - //! Disable operator= - GslResourceReference& operator=(const GslResourceReference&); -}; - -//! GPU resource -class Resource : public amd::HeapObject { - public: - enum InteropType { - InteropTypeless = 0, - InteropVertexBuffer, - InteropIndexBuffer, - InteropRenderBuffer, - InteropTexture, - InteropTextureViewLevel, - InteropTextureViewCube, - InteropSurface - }; - - struct CreateParams : public amd::StackObject { - amd::Memory* owner_; //!< Resource's owner - VirtualGPU* gpu_; //!< Resource won't be shared between multiple queues - CreateParams() : owner_(NULL), gpu_(NULL) {} - }; - - struct PinnedParams : public CreateParams { - const amd::HostMemoryReference* hostMemRef_; //!< System memory pointer for pinning - size_t size_; //!< System memory size - }; - - struct ViewParams : public CreateParams { - size_t offset_; //!< Alias resource offset - size_t size_; //!< Alias resource size - const Resource* resource_; //!< Parent resource for the view creation - const void* memory_; - }; - - struct ImageViewParams : public CreateParams { - size_t level_; //!< Image mip level for a new view - size_t layer_; //!< Image layer for a new view - const Resource* resource_; //!< Parent resource for the view creation - const void* memory_; - }; - - struct ImageBufferParams : public CreateParams { - const Resource* resource_; //!< Parent resource for the image creation - const void* memory_; - }; - - struct OGLInteropParams : public CreateParams { - InteropType type_; //!< OGL resource type - CALuint handle_; //!< OGL resource handle - uint mipLevel_; //!< Texture mip level - uint layer_; //!< Texture layer - void* glPlatformContext_; - void* glDeviceContext_; - uint flags_; - }; - -#ifdef _WIN32 - struct D3DInteropParams : public CreateParams { - InteropType type_; //!< D3D resource type - void* iDirect3D_; //!< D3D resource interface object - HANDLE handle_; //!< D3D resource handle - uint mipLevel_; //!< Texture mip level - int layer_; //!< Texture layer - uint misc; //!< miscellaneous cases - }; -#endif // _WIN32 - - //! Resource memory - enum MemoryType { - Empty = 0x0, //!< resource is empty - Local, //!< resource in local memory - Persistent, //!< resource in persistent memory - Remote, //!< resource in nonlocal memory - RemoteUSWC, //!< resource in nonlocal memory - Pinned, //!< resource in pinned system memory - View, //!< resource is an alias - OGLInterop, //!< resource is an OGL memory object - D3D10Interop, //!< resource is a D3D10 memory object - D3D11Interop, //!< resource is a D3D11 memory object - Heap, //!< resource is a heap - ImageView, //!< resource is a view to some image - ImageBuffer, //!< resource is an image view of a buffer - BusAddressable, //!< resource is a bus addressable memory - ExternalPhysical, //!< resource is an external physical memory - D3D9Interop, //!< resource is a D3D9 memory object - Scratch, //!< resource is scratch memory - Shader, //!< resource is a shader - }; - - //! Resource map flags - enum MapFlags { - Discard = 0x00000001, //!< discard lock - NoOverwrite = 0x00000002, //!< lock with no overwrite - ReadOnly = 0x00000004, //!< lock for read only operation - WriteOnly = 0x00000008, //!< lock for write only operation - NoWait = 0x00000010, //!< lock with no wait - }; - - //! CAL resource descriptor - struct CalResourceDesc : public amd::HeapObject { - MemoryType type_; //!< Memory type - size_t width_; //!< CAL resource width - size_t height_; //!< CAL resource height - size_t depth_; //!< CAL resource depth - uint mipLevels_; //!< Number of mip levels - cmSurfFmt format_; //!< GSL resource format - CALuint flags_; //!< CAL resource flags, used in creation - size_t pitch_; //!< CAL resource pitch, valid if locked - CALuint slice_; //!< CAL resource slice, valid if locked - gslChannelOrder channelOrder_; //!< GSL resource channel order - gslMemObjectAttribType dimension_; //!< GSL resource dimension - cl_mem_object_type imageType_; //!< CL image type - union { - struct { - uint dimSize_ : 2; //!< Dimension size - uint cardMemory_ : 1; //!< GSL resource is in video memory - uint imageArray_ : 1; //!< GSL resource is an array of images - uint buffer_ : 1; //!< GSL resource is a buffer - uint tiled_ : 1; //!< GSL resource is tiled - uint scratch_ : 1; //!< Scratch buffer - uint skipRsrcCache_ : 1; //!< Skip caching of a cal resource - uint isAllocSVM_ : 1; //!< SVM resource attribute - uint isAllocExecute_ : 1; //!< SVM resource allocation attribute for shader\cmdbuf - }; - uint state_; - }; - }; - - //! Constructor of 1D Resource object - Resource(const Device& gpuDev, //!< GPU device object - size_t width, //!< resource width - cmSurfFmt format //!< resource format - ); - - //! Constructor of Image Resource object - Resource(const Device& gpuDev, //!< GPU device object - size_t width, //!< resource width - size_t height, //!< resource height - size_t depth, //!< resource depth - cmSurfFmt format, //!< resource format - gslChannelOrder chOrder, //!< resource channel order - cl_mem_object_type imageType, //!< CL image type - uint mipLevels = 1 //!< Number of mip levels - ); - - //! Destructor of the resource - virtual ~Resource(); - - /*! \brief Creates a CAL object, associated with the resource - * - * \return True if we succesfully created a CAL resource - */ - virtual bool create(MemoryType memType, //!< memory type - CreateParams* params = 0 //!< special parameters for resource allocation - ); - - /*! \brief Copies a subregion of memory from one resource to another - * - * This is a general copy from anything to anything (as long as it fits). - * All positions and sizes are given in bytes. Note, however, that only - * a subset of this general interface is currently implemented. - * - * \return true if successful - */ - bool partialMemCopyTo(VirtualGPU& gpu, //!< Virtual GPU device object - const amd::Coord3D& srcOrigin, //!< Origin of the source region - const amd::Coord3D& dstOrigin, //!< Origin of the destination region - const amd::Coord3D& size, //!< Size of the region to copy - Resource& dstResource, //!< Destination resource - bool enableRectCopy = false, //!< Rectangular DMA support - bool flushDMA = false, //!< Flush DMA if requested - uint bytesPerElement = 1 //!< Bytes Per Element - ) const; - - /*! \brief Copies size/4 DWORD of memory to a surface - * - * This is a raw copy to any surface using a CP packet. - * Size needs to be atleast a DWORD or multiple - * - */ - void writeRawData(VirtualGPU& gpu, //!< Virtual GPU device object - size_t size, //!< Size in bytes of data to be copied(multiple of DWORDS) - const void* data, //!< Data to be copied - bool waitForEvent //!< Wait for event complete - ) const; - - //! Returns the offset in GPU memory for aliases - size_t offset() const { return offset_; } - - //! Returns the offset in GPU heap - uint64_t hbOffset() const { return hbOffset_; } - - //! Returns the pinned memory offset - uint64_t pinOffset() const { return pinOffset_; } - - //! Returns the size in GPU heap - uint64_t hbSize() const { return hbSize_; } - - //! Returns the GPU device that owns this resource - const Device& dev() const { return gpuDevice_; } - - //! Returns the CAL descriptor for resource - const CalResourceDesc* cal() const { return &cal_; } - - //! Returns the CAL resource handle - gslMemObject gslResource() const { return gslRef_->gslResource(); } - - //! Returns global memory offset - uint64_t vmAddress() const { return gslResource()->getSurfaceAddress(); } - - //! Returns global memory offset - bool mipMapped() const { return (cal()->mipLevels_ > 1) ? true : false; } - - //! Checks if persistent memory can have a direct map - bool isPersistentDirectMap() const; - - /*! \brief Locks the resource and returns a physical pointer - * - * \note This operation stalls HW pipeline! - * - * \return Pointer to the physical memory - */ - void* map(VirtualGPU* gpu, //!< Virtual GPU device object - uint flags = 0, //!< flags for the map operation - // Optimization for multilayer map/unmap - uint startLayer = 0, //!< Start layer for multilayer map - uint numLayers = 0 //!< End layer for multilayer map - ); - - //! Unlocks the resource if it was locked - void unmap(VirtualGPU* gpu //!< Virtual GPU device object - ); - - //! Marks the resource as busy - void setBusy(VirtualGPU& gpu, //!< Virtual GPU device object - GpuEvent calEvent //!< CAL event - ) const; - - //! Wait for the resource - void wait(VirtualGPU& gpu, //!< Virtual GPU device object - bool waitOnBusyEngine = false //!< Wait only if engine has changed - ) const; - - //! Performs host write to the resource GPU memory - bool hostWrite(VirtualGPU* gpu, //!< Virtual GPU device object - const void* hostPtr, //!< Host pointer to the SRC data - const amd::Coord3D& origin, //!< Offsets for the update - const amd::Coord3D& size, //!< The number of bytes to write - uint flags = 0, //!< Map flags - size_t rowPitch = 0, //!< Raw data row pitch - size_t slicePitch = 0 //!< Raw data slice pitch - ); - - //! Performs host read from the resource GPU memory - bool hostRead(VirtualGPU* gpu, //!< Virtual GPU device object - void* hostPtr, //!< Host pointer to the DST data - const amd::Coord3D& origin, //!< Offsets for the update - const amd::Coord3D& size, //!< The number of bytes to write - size_t rowPitch = 0, //!< Raw data row pitch - size_t slicePitch = 0 //!< Raw data slice pitch - ); - - //! Warms up the rename list for this resource - void warmUpRenames(VirtualGPU& gpu); - - //! Gets the resource element size - size_t elementSize() const { return elementSize_; } - - //! Get the mapped address of this resource - address data() const { return reinterpret_cast
(address_); } - - //! Frees all allocated CAL memories and resources, - //! associated with this objects. And also destroys all rename structures - //! Note: doesn't destroy the object itself - void free(); - - //! Return memory type - MemoryType memoryType() const { return cal_.type_; } - - //! Retunrs true if memory type matches specified - bool isMemoryType(MemoryType memType) const; - - //! Returns TRUE if resource was allocated as cacheable - bool isCacheable() const { return (isMemoryType(Remote) || isMemoryType(Pinned)) ? true : false; } - - bool gslGLAcquire(); - bool gslGLRelease(); - - //! Returns HW state for the resource (used for images only) - const void* hwState() const { return hwState_; } - - //! Returns CPU HW SRD for the resource (used for images only) - uint64_t hwSrd() const { return hwSrd_; } - - protected: - size_t elementSize_; //!< Size of a single element in bytes - - private: - //! Disable copy constructor - Resource(const Resource&); - - //! Disable operator= - Resource& operator=(const Resource&); - - typedef std::vector RenameList; - - //! Rename current resource - bool rename(VirtualGPU& gpu, //!< Virtual GPU device object - bool force = false //!< Force renaming - ); - - //! Sets the rename as active - void setActiveRename(VirtualGPU& gpu, //!< Virtual GPU device object - GslResourceReference* rename //!< new active rename - ); - - //! Gets the active rename - bool getActiveRename(VirtualGPU& gpu, //!< Virtual GPU device object - GslResourceReference** rename //!< Saved active rename - ); - - /*! \brief Locks the resource with layers and returns a physical pointer - * - * \return Pointer to the physical memory - */ - void* mapLayers(VirtualGPU* gpu, //!< Virtual GPU device object - CALuint flags = 0 //!< flags for the map operation - ); - - //! Unlocks the resource with layers if it was locked - void unmapLayers(VirtualGPU* gpu //!< Virtual GPU device object - ); - - //! Calls GSL to map a resource - void* gslMap(size_t* pitch, //!< Pitch value for the image - gslMapAccessType flags, //!< Map flags - gslMemObject resource //!< GSL memory object - ) const; - - //! Uses GSL to unmap a resource - void gslUnmap(gslMemObject resource //!< GSL memory object - ) const; - - //! Fress all GSL resources associated with OCL resource - void gslFree() const; - - const Device& gpuDevice_; //!< GPU device - CalResourceDesc cal_; //!< CAL descriptor for this resource - std::atomic mapCount_; //!< Total number of maps - void* address_; //!< Physical address of this resource - size_t offset_; //!< Resource offset - size_t curRename_; //!< Current active rename in the list - RenameList renames_; //!< Rename resource list - GslResourceReference* gslRef_; //!< GSL resource reference - const Resource* viewOwner_; //!< GPU resource, which owns this view - uint64_t hbOffset_; //!< Offset in the heap (virtual or real) - uint64_t hbSize_; //!< Memory size - uint64_t pinOffset_; //!< Pinned memory offset - gslMemObject glInterop_; //!< Original GL interop object - void* glInteropMbRes_; //!< Mb Res handle - uint32_t glType_; //!< GL interop type - void* glPlatformContext_; - void* glDeviceContext_; - - // Optimization for multilayer map/unmap - uint startLayer_; //!< Start layer for map/unmapLayer - uint numLayers_; //!< Number of layers for map/unmapLayer - CALuint mapFlags_; //!< Map flags for map/umapLayer - - //! @note: This field is necessary for the thread safe release only - VirtualGPU* gpu_; //!< Resource will be used only on this queue - - uint32_t* hwState_; //!< HW state for image object - uint64_t hwSrd_; //!< GPU pointer to HW SRD -}; - -class ResourceCache : public amd::HeapObject { - public: - //! Default constructor - ResourceCache(size_t cacheSizeLimit) - : lockCacheOps_("CAL resource cache", true) - , cacheSize_(0) - , lclCacheSize_(0) - , cacheSizeLimit_(cacheSizeLimit) {} - - //! Default destructor - ~ResourceCache(); - - //! Adds a CAL resource to the cache - bool addCalResource(Resource::CalResourceDesc* desc, //!< CAL resource descriptor - cache key - GslResourceReference* ref //!< CAL resource reference - ); - - //! Finds a CAL resource from the cache - GslResourceReference* findCalResource( - Resource::CalResourceDesc* desc //!< CAL resource descriptor - cache key - ); - - //! Destroys cache - bool free(size_t minCacheEntries = 0); - - //! Returns the size of all memory, stored in the cache - size_t cacheSize() const { return cacheSize_; } - - //! Returns the size of local memory, stored in the cache - size_t lclCacheSize() const { return lclCacheSize_; } - - private: - //! Disable copy constructor - ResourceCache(const ResourceCache&); - - //! Disable operator= - ResourceCache& operator=(const ResourceCache&); - - //! Gets resource size in bytes - size_t getResourceSize(Resource::CalResourceDesc* desc); - - //! Removes one last entry from the cache - void removeLast(); - - amd::Monitor lockCacheOps_; //!< Lock to serialise cache access - - size_t cacheSize_; //!< Current cache size in bytes - size_t lclCacheSize_; //!< Local memory stored in the cache - size_t cacheSizeLimit_; //!< Cache size limit in bytes - - //! CAL resource cache - std::list > resCache_; -}; - -/*@}*/} // namespace gpu - -#endif /*GPURESOURCE_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gpusched.hpp b/projects/clr/rocclr/device/gpu/gpusched.hpp deleted file mode 100644 index 2ebc9e8750..0000000000 --- a/projects/clr/rocclr/device/gpu/gpusched.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUSCHED_HPP_ -#define GPUSCHED_HPP_ - -#include "hsa.h" - -namespace gpu { - -//! AmdAqlWrap slot state -enum AqlWrapState { - AQL_WRAP_FREE = 0, - AQL_WRAP_RESERVED, - AQL_WRAP_READY, - AQL_WRAP_MARKER, - AQL_WRAP_BUSY, - AQL_WRAP_DONE -}; - -struct AmdVQueueHeader { - uint32_t aql_slot_num; //!< [LRO/SRO] The total number of the AQL slots (multiple of 64). - uint32_t event_slot_num; //!< [LRO] The number of kernel events in the events buffer - uint64_t event_slot_mask; //!< [LRO] A pointer to the allocation bitmask array for the events - uint64_t event_slots; //!< [LRO] Pointer to a buffer for the events. - // Array of event_slot_num entries of AmdEvent - uint64_t aql_slot_mask; //!< [LRO/SRO]A pointer to the allocation bitmask for aql_warp slots - uint32_t command_counter; //!< [LRW] The global counter for the submitted commands into the queue - uint32_t wait_size; //!< [LRO] The wait list size (in clk_event_t) - uint32_t arg_size; //!< [LRO] The size of argument buffer (in bytes) - uint32_t mask_groups; //!< Processed mask groups by one thread - uint64_t - kernel_table; //!< [LRO] Pointer to an array with all kernel objects (ulong for each entry) - uint32_t reserved[2]; //!< For the future usage -}; - -struct AmdAqlWrap { - uint32_t state; //!< [LRW/SRW] The current state of the AQL wrapper: FREE, RESERVED, READY, - // MARKER, BUSY and DONE. The block could be returned back to a free state. - uint32_t enqueue_flags; //!< [LWO/SRO] Contains the flags for the kernel execution start - uint32_t command_id; //!< [LWO/SRO] The unique command ID - uint32_t child_counter; //!< [LRW/SRW] Counter that determine the launches of child kernels. - // It’s incremented on the - // start and decremented on the finish. The parent kernel can be considered as - // done when the value is 0 and the state is DONE - uint64_t completion; //!< [LWO/SRO] CL event for the current execution (clk_event_t) - uint64_t parent_wrap; //!< [LWO/SRO] Pointer to the parent AQL wrapper (AmdAqlWrap*) - uint64_t wait_list; //!< [LRO/SRO] Pointer to an array of clk_event_t objects (64 bytes default) - uint32_t wait_num; //!< [LWO/SRO] The number of cl_event_wait objects - uint32_t reserved[5]; //!< For the future usage - hsa_kernel_dispatch_packet_t aql; //!< [LWO/SRO] AQL packet – 64 bytes AQL packet -}; - -struct AmdEvent { - uint32_t state; //!< [LRO/SRW] Event state: START, END, COMPLETE - uint32_t counter; //!< [LRW] Event retain/release counter. 0 means the event is free - uint64_t timer[3]; //!< [LRO/SWO] Timer values for profiling for each state - uint64_t captureInfo; //!< [LRW/SRO] Profiling capture info for CLK_PROFILING_COMMAND_EXEC_TIME -}; - -struct SchedulerParam { - uint32_t signal; //!< Signal to stop the child queue(address must be 16 bytes aligned) - uint32_t eng_clk; //!< Engine clock in Mhz - uint64_t hw_queue; //!< Address to HW queue - uint64_t hsa_queue; //!< Address to HSA dummy queue - uint32_t useATC; //!< GPU access to shader program by ATC. - uint32_t scratchSize; //!< Scratch buffer size - uint64_t scratch; //!< GPU address to the scratch buffer - uint32_t numMaxWaves; //!< The max number of possible waves - uint32_t releaseHostCP; //!< Releases CP on the host queue - uint64_t parentAQL; //!< Host parent AmdAqlWrap packet - uint32_t dedicatedQueue; //!< Scheduler uses a dedicated queue - uint32_t scratchOffset; //!< Scratch buffer offset - uint32_t reserved[2]; //!< Reserved -}; - -} // namespace gpu - -#endif diff --git a/projects/clr/rocclr/device/gpu/gpuschedcl.cpp b/projects/clr/rocclr/device/gpu/gpuschedcl.cpp deleted file mode 100644 index 55f9394d57..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuschedcl.cpp +++ /dev/null @@ -1,310 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -namespace gpu { - -#define SCHEDULER_KERNEL(...) #__VA_ARGS__ - -const char* SchedulerSourceCode = SCHEDULER_KERNEL( -\n -extern void __amd_scheduler(__global void *, __global void *, uint); -\n -typedef struct _HsaAqlDispatchPacket { - uint mix; - ushort workgroup_size[3]; - ushort reserved2; - uint grid_size[3]; - uint private_segment_size_bytes; - uint group_segment_size_bytes; - ulong kernel_object_address; - ulong kernel_arg_address; - ulong reserved3; - ulong completion_signal; -} HsaAqlDispatchPacket; -\n -// This is an OpenCLized hsa_control_directives_t -typedef struct _AmdControlDirectives { - ulong enabled_control_directives; - ushort enable_break_exceptions; - ushort enable_detect_exceptions; - uint max_dynamic_group_size; - ulong max_flat_grid_size; - uint max_flat_workgroup_size; - uchar required_dim; - uchar reserved1[3]; - ulong required_grid_size[3]; - uint required_workgroup_size[3]; - uchar reserved2[60]; -} AmdControlDirectives; -\n -// This is an OpenCLized amd_kernel_code_t -typedef struct _AmdKernelCode { - uint amd_kernel_code_version_major; - uint amd_kernel_code_version_minor; - ushort amd_machine_kind; - ushort amd_machine_version_major; - ushort amd_machine_version_minor; - ushort amd_machine_version_stepping; - long kernel_code_entry_byte_offset; - long kernel_code_prefetch_byte_offset; - ulong kernel_code_prefetch_byte_size; - ulong max_scratch_backing_memory_byte_size; - uint compute_pgm_rsrc1; - uint compute_pgm_rsrc2; - uint kernel_code_properties; - uint workitem_private_segment_byte_size; - uint workgroup_group_segment_byte_size; - uint gds_segment_byte_size; - ulong kernarg_segment_byte_size; - uint workgroup_fbarrier_count; - ushort wavefront_sgpr_count; - ushort workitem_vgpr_count; - ushort reserved_vgpr_first; - ushort reserved_vgpr_count; - ushort reserved_sgpr_first; - ushort reserved_sgpr_count; - ushort debug_wavefront_private_segment_offset_sgpr; - ushort debug_private_segment_buffer_sgpr; - uchar kernarg_segment_alignment; - uchar group_segment_alignment; - uchar private_segment_alignment; - uchar wavefront_size; - int call_convention; - uchar reserved1[12]; - ulong runtime_loader_kernel_symbol; - AmdControlDirectives control_directives; -} AmdKernelCode; -\n -typedef struct _HwDispatchHeader { - uint writeData0; // CP WRITE_DATA write to rewind for memory - uint writeData1; - uint writeData2; - uint writeData3; - uint rewind; // REWIND execution - uint startExe; // valid bit - uint condExe0; // 0xC0032200 -- TYPE 3, COND_EXEC - uint condExe1; // 0x00000204 ---- - uint condExe2; // 0x00000000 ---- - uint condExe3; // 0x00000000 ---- - uint condExe4; // 0x00000000 ---- -} HwDispatchHeader; -\n -typedef struct _HwDispatch { - uint packet0; // 0xC0067602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (6 values) - uint offset0; // 0x00000204 ---- OFFSET - uint startX; // 0x00000000 ---- COMPUTE_START_X: START = 0x0 - uint startY; // 0x00000000 ---- COMPUTE_START_Y: START = 0x0 - uint startZ; // 0x00000000 ---- COMPUTE_START_Z: START = 0x0 - uint wrkGrpSizeX; // 0x00000000 ---- COMPUTE_NUM_THREAD_X: NUM_THREAD_FULL = 0x0, NUM_THREAD_PARTIAL = 0x0 - uint wrkGrpSizeY; // 0x00000000 ---- COMPUTE_NUM_THREAD_Y: NUM_THREAD_FULL = 0x0, NUM_THREAD_PARTIAL = 0x0 - uint wrkGrpSizeZ; // 0x00000000 ---- COMPUTE_NUM_THREAD_Z: NUM_THREAD_FULL = 0x0, NUM_THREAD_PARTIAL = 0x0 - uint packet1; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values) - uint offset1; // 0x0000020C ---- OFFSET - uint isaLo; // 0x00000000 ---- COMPUTE_PGM_LO: DATA = 0x0 - uint isaHi; // 0x00000000 ---- COMPUTE_PGM_HI: DATA = 0x0, INST_ATC__CI__VI = 0x0 - uint packet2; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values) - uint offset2; // 0x00000212 ---- OFFSET - uint resource1; // 0x00000000 ---- COMPUTE_PGM_RSRC1 - uint resource2; // 0x00000000 ---- COMPUTE_PGM_RSRC2 - uint packet3; // 0xc0017602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (1 value) - uint offset3; // 0x00000215 ---- OFFSET - uint pad31; // 0x000003ff ---- COMPUTE_RESOURCE_LIMITS - uint packet31; // 0xC0067602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (1 value) - uint offset31; // 0x00000218 ---- OFFSET - uint ringSize; // 0x00000000 ---- COMPUTE_TMPRING_SIZE: WAVES = 0x0, WAVESIZE = 0x0 - uint user0; // 0xC0047602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (4 values) - uint offsUser0; // 0x00000240 ---- OFFSET - uint scratchLo; // 0x00000000 ---- COMPUTE_USER_DATA_0: DATA = 0x0 - uint scratchHi; // 0x80000000 ---- COMPUTE_USER_DATA_1: DATA = 0x80000000 - uint scratchSize; // 0x00000000 ---- COMPUTE_USER_DATA_2: DATA = 0x0 - uint padUser; // 0x00EA7FAC ---- COMPUTE_USER_DATA_3: DATA = 0xEA7FAC - uint user1; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values) - uint offsUser1; // 0x00000244 ---- OFFSET - uint aqlPtrLo; // 0x00000000 ---- COMPUTE_USER_DATA_4: DATA = 0x0 - uint aqlPtrHi; // 0x00000000 ---- COMPUTE_USER_DATA_5: DATA = 0x0 - uint user2; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values) - uint offsUser2; // 0x00000246 ---- OFFSET - uint hsaQueueLo; // 0x00000000 ---- COMPUTE_USER_DATA_6: DATA = 0x0 - uint hsaQueueHi; // 0x00000000 ---- COMPUTE_USER_DATA_7: DATA = 0x0 - uint user3; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values) - uint offsUser3; // 0x00000246 ---- OFFSET - uint argsLo; // 0x00000000 ---- COMPUTE_USER_DATA_8: DATA = 0x0 - uint argsHi; // 0x00000000 ---- COMPUTE_USER_DATA_9: DATA = 0x0 - uint copyData; // 0xC0044000 -- TYPE 3, COPY_DATA - uint copyDataFlags; // 0x00000405 ---- srcSel 0x5, destSel 0x4, countSel 0x0, wrConfirm 0x0, engineSel 0x0 - uint scratchAddrLo; // 0x000201C4 ---- srcAddressLo - uint scratchAddrHi; // 0x00000000 ---- srcAddressHi - uint shPrivateLo; // 0x00002580 ---- dstAddressLo - uint shPrivateHi; // 0x00000000 ---- dstAddressHi - uint user4; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values) - uint offsUser4; // 0x00000248 ---- OFFSET - uint scratchOffs; // 0x00000000 ---- COMPUTE_USER_DATA_10: DATA = 0x0 - uint privSize; // 0x00000030 ---- COMPUTE_USER_DATA_11: DATA = 0x30 - uint packet4; // 0xC0031502 -- TYPE 3, DISPATCH_DIRECT, TYPE:COMPUTE - uint glbSizeX; // 0x00000000 - uint glbSizeY; // 0x00000000 - uint glbSizeZ; // 0x00000000 - uint padd41; // 0x00000021 -} HwDispatch; -\n -static const uint WavefrontSize = 64; -static const uint MaxWaveSize = 0x400; -static const uint UsrRegOffset = 0x240; -static const uint Pm4Nop = 0xC0001002; -static const uint Pm4UserRegs = 0xC0007602; -static const uint Pm4CopyReg = 0xC0044000; -static const uint PrivateSegEna = 0x1; -static const uint DispatchEna = 0x2; -static const uint QueuePtrEna = 0x4; -static const uint KernelArgEna = 0x8; -static const uint FlatScratchEna = 0x20; -\n -uint GetCmdTemplateHeaderSize() { return sizeof(HwDispatchHeader); } -\n -uint GetCmdTemplateDispatchSize() { return sizeof(HwDispatch); } -\n -void EmptyCmdTemplateDispatch(ulong cmdBuf) -{ - volatile __global HwDispatch* dispatch = (volatile __global HwDispatch*)cmdBuf; - dispatch->glbSizeX = 0; - dispatch->glbSizeY = 0; - dispatch->glbSizeZ = 0; -} -\n -void RunCmdTemplateDispatch( - ulong cmdBuf, - __global HsaAqlDispatchPacket* aqlPkt, - ulong scratch, - ulong hsaQueue, - uint scratchSize, - uint scratchOffset, - uint numMaxWaves, - uint useATC) -\n -{ - volatile __global HwDispatch* dispatch = (volatile __global HwDispatch*)cmdBuf; - uint usrRegCnt = 0; - - // Program workgroup size - dispatch->wrkGrpSizeX = aqlPkt->workgroup_size[0]; - dispatch->wrkGrpSizeY = aqlPkt->workgroup_size[1]; - dispatch->wrkGrpSizeZ = aqlPkt->workgroup_size[2]; - - // ISA address - __global AmdKernelCode* kernelObj = (__global AmdKernelCode*)aqlPkt->kernel_object_address; - ulong isa = aqlPkt->kernel_object_address + kernelObj->kernel_code_entry_byte_offset; - - dispatch->isaLo = (uint)(isa >> 8); - dispatch->isaHi = (uint)(isa >> 40) | (useATC ? 0x100 : 0); - - // Program PGM resource registers - dispatch->resource1 = kernelObj->compute_pgm_rsrc1; - dispatch->resource2 = kernelObj->compute_pgm_rsrc2; - - uint flags = kernelObj->kernel_code_properties; - uint privateSize = kernelObj->workitem_private_segment_byte_size; - - uint ldsSize = aqlPkt->group_segment_size_bytes; - - // Align up the LDS blocks 128 * 4(in DWORDs) - uint ldsBlocks = (ldsSize + 511) >> 9; - - dispatch->resource2 |= (ldsBlocks << 15); - - // Private/scratch segment was enabled - if (flags & PrivateSegEna) { - uint waveSize = privateSize * WavefrontSize; - // 256 DWRODs is the minimum for SQ - waveSize = max(MaxWaveSize, waveSize); - - uint numWaves = scratchSize / waveSize; - - numWaves = min(numWaves, numMaxWaves); - - dispatch->ringSize = numWaves; - dispatch->ringSize |= (waveSize >> 10) << 12; - dispatch->user0 = Pm4UserRegs | (4 << 16); - dispatch->scratchLo = (uint)scratch; - dispatch->scratchHi = ((uint)(scratch >> 32)) | 0x80000000; // Enables swizzle - dispatch->scratchSize = scratchSize; - usrRegCnt += 4; - } - else { - dispatch->ringSize = 0; - dispatch->user0 = Pm4Nop | (4 << 16); - } - - // Pointer to the AQL dispatch packet - dispatch->user1 = (flags & DispatchEna) ? (Pm4UserRegs | (2 << 16)) : (Pm4Nop | (2 << 16)); - dispatch->offsUser1 = UsrRegOffset + usrRegCnt; - usrRegCnt += (flags & DispatchEna) ? 2 : 0; - ulong gpuAqlPtr = (ulong)aqlPkt; - dispatch->aqlPtrLo = (uint)gpuAqlPtr; - dispatch->aqlPtrHi = (uint)(gpuAqlPtr >> 32); - - // Pointer to the AQL queue header - if (flags & QueuePtrEna) { - dispatch->user2 = Pm4UserRegs | (2 << 16); - dispatch->offsUser2 = UsrRegOffset + usrRegCnt; - usrRegCnt += 2; - dispatch->hsaQueueLo = (uint)hsaQueue; - dispatch->hsaQueueHi = (uint)(hsaQueue >> 32); - } - else { - dispatch->user2 = Pm4Nop | (2 << 16); - } - - // Pointer to the AQL kernel arguments - dispatch->user3 = (flags & KernelArgEna) ? (Pm4UserRegs | (2 << 16)) : (Pm4Nop | (2 << 16)); - dispatch->offsUser3 = UsrRegOffset + usrRegCnt; - usrRegCnt += (flags & KernelArgEna) ? 2 : 0; - dispatch->argsLo = (uint)aqlPkt->kernel_arg_address; - dispatch->argsHi = (uint)(aqlPkt->kernel_arg_address >> 32); - - // Provide pointer to the private/scratch buffer for the flat address - if (flags & FlatScratchEna) { - dispatch->copyData = Pm4CopyReg; - dispatch->scratchAddrLo = (uint)((scratch - scratchOffset) >> 16); - dispatch->offsUser4 = UsrRegOffset + usrRegCnt; - dispatch->scratchOffs = scratchOffset; - dispatch->privSize = privateSize; - } - else { - dispatch->copyData = Pm4Nop | (8 << 16); - } - - // Update the global launch grid - dispatch->glbSizeX = aqlPkt->grid_size[0]; - dispatch->glbSizeY = aqlPkt->grid_size[1]; - dispatch->glbSizeZ = aqlPkt->grid_size[2]; -} -\n -__kernel void -__amd_rocclr_scheduler( - __global void * queue, - __global void * params, - uint paramIdx) -{ - __amd_scheduler(queue, params, paramIdx); -} -\n -); - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuscsi.cpp b/projects/clr/rocclr/device/gpu/gpuscsi.cpp deleted file mode 100644 index a07122af2f..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuscsi.cpp +++ /dev/null @@ -1,210 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpukernel.hpp" -#include "hsailctx.hpp" -#include "SCShadersSi.h" -#include "si_ci_vi_merged_offset.h" -#include "si_ci_vi_merged_registers.h" -#include -#include -#include -#include -#include -#include "amd_hsa_loader.hpp" - -namespace gpu { - -bool NullKernel::siCreateHwInfo(const void* shader, AMUabiAddEncoding& encoding) { - static const uint NumSiCsInfos = (70 + 5 + 1 + 32 + 6); - CALProgramInfoEntry* newInfos; - uint i = 0; - uint infoCount = NumSiCsInfos; - const SC_SI_HWSHADER_CS* cShader = reinterpret_cast(shader); - newInfos = new CALProgramInfoEntry[infoCount]; - encoding.progInfos = newInfos; - if (encoding.progInfos == 0) { - infoCount = 0; - return false; - } - newInfos[i].address = AMU_ABI_USER_ELEMENT_COUNT; - newInfos[i].value = cShader->common.userElementCount; - i++; - for (unsigned int j = 0; j < cShader->common.userElementCount; j++) { - newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD0 + 4 * j; - newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].dataClass; - i++; - newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD1 + 4 * j; - newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].apiSlot; - i++; - newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD2 + 4 * j; - newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].startUserReg; - i++; - newInfos[i].address = AMU_ABI_USER_ELEMENTS_0_DWORD3 + 4 * j; - newInfos[i].value = HWSHADER_Get(cShader, common.pUserElements)[j].userRegCount; - i++; - } - - newInfos[i].address = AMU_ABI_SI_NUM_VGPRS; - newInfos[i].value = cShader->common.numVgprs; - i++; - newInfos[i].address = AMU_ABI_SI_NUM_SGPRS; - newInfos[i].value = cShader->common.numSgprs; - i++; - newInfos[i].address = AMU_ABI_SI_NUM_SGPRS_AVAIL; - newInfos[i].value = SI_sgprs_avail; // 512;//options.NumSGPRsAvailable; - i++; - newInfos[i].address = AMU_ABI_SI_NUM_VGPRS_AVAIL; - newInfos[i].value = SI_vgprs_avail; // options.NumVGPRsAvailable; - i++; - - newInfos[i].address = AMU_ABI_SI_FLOAT_MODE; - newInfos[i].value = cShader->common.floatMode; - i++; - newInfos[i].address = AMU_ABI_SI_IEEE_MODE; - newInfos[i].value = cShader->common.bIeeeMode; - i++; - - newInfos[i].address = AMU_ABI_SI_SCRATCH_SIZE; - newInfos[i].value = cShader->common.scratchSize; - ; - i++; - - newInfos[i].address = mmCOMPUTE_PGM_RSRC2; - newInfos[i].value = cShader->computePgmRsrc2.u32All; - i++; - - newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_X; - newInfos[i].value = cShader->numThreadX; - i++; - newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_Y; - newInfos[i].value = cShader->numThreadY; - i++; - newInfos[i].address = AMU_ABI_NUM_THREAD_PER_GROUP_Z; - newInfos[i].value = cShader->numThreadZ; - i++; - - newInfos[i].address = AMU_ABI_ORDERED_APPEND_ENABLE; - newInfos[i].value = cShader->bOrderedAppendEnable; - i++; - - newInfos[i].address = AMU_ABI_RAT_OP_IS_USED; - newInfos[i].value = cShader->common.uavResourceUsage[0]; - i++; - - for (unsigned int j = 0; j < ((SC_MAX_UAV + 31) / 32); j++) { - newInfos[i].address = AMU_ABI_UAV_RESOURCE_MASK_0 + j; - newInfos[i].value = cShader->common.uavResourceUsage[j]; - i++; - } - - newInfos[i].address = AMU_ABI_NUM_WAVEFRONT_PER_SIMD; // Setting the same as for scWrapR800Info - newInfos[i].value = 1; - i++; - - newInfos[i].address = AMU_ABI_WAVEFRONT_SIZE; - newInfos[i].value = nullDev().isa().simdWidth() * 4; // options.WavefrontSize; - i++; - - newInfos[i].address = AMU_ABI_LDS_SIZE_AVAIL; - newInfos[i].value = SI_ldssize_avail; // options.LDSSize; - i++; - - COMPUTE_PGM_RSRC2 computePgmRsrc2; - computePgmRsrc2.u32All = cShader->computePgmRsrc2.u32All; - - newInfos[i].address = AMU_ABI_LDS_SIZE_USED; - newInfos[i].value = 64 * 4 * computePgmRsrc2.bits.LDS_SIZE; - i++; - - infoCount = i; - assert((i + 4 * (16 - cShader->common.userElementCount)) == NumSiCsInfos); - encoding.progInfosCount = infoCount; - - encoding.textData = HWSHADER_Get(cShader, common.hShaderMemHandle); - encoding.textSize = cShader->common.codeLenInByte; - instructionCnt_ = encoding.textSize / sizeof(uint32_t); - encoding.scratchRegisterCount = cShader->common.scratchSize; - encoding.UAVReturnBufferTotalSize = 0; - - return true; -} - -bool HSAILKernel::aqlCreateHWInfo(amd::hsa::loader::Symbol* sym) { - if (!sym) { - return false; - } - uint64_t akc_addr = 0; - if (!sym->GetInfo(HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, reinterpret_cast(&akc_addr))) { - return false; - } - amd_kernel_code_t* akc = reinterpret_cast(akc_addr); - cpuAqlCode_ = akc; - if (!sym->GetInfo(HSA_EXT_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT_SIZE, - reinterpret_cast(&codeSize_))) { - return false; - } - size_t akc_align = 0; - if (!sym->GetInfo(HSA_EXT_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT_ALIGN, - reinterpret_cast(&akc_align))) { - return false; - } - - // Allocate HW resources for the real program only - if (!prog().isNull()) { - code_ = new gpu::Memory(dev(), amd::alignUp(codeSize_, akc_align)); - // Initialize kernel ISA code - if (code_ && code_->create(Resource::Shader)) { - address cpuCodePtr = static_cast
(code_->map(NULL, Resource::WriteOnly)); - // Copy only amd_kernel_code_t - memcpy(cpuCodePtr, reinterpret_cast
(akc), codeSize_); - code_->unmap(NULL); - } else { - LogError("Failed to allocate ISA code!"); - return false; - } - } - - assert((akc->workitem_private_segment_byte_size & 3) == 0 && "Scratch must be DWORD aligned"); - workGroupInfo_.scratchRegs_ = - amd::alignUp(akc->workitem_private_segment_byte_size, 16) / sizeof(uint); - workGroupInfo_.privateMemSize_ = akc->workitem_private_segment_byte_size; - workGroupInfo_.availableLDSSize_ = dev().info().localMemSize_; - workGroupInfo_.localMemSize_ = workGroupInfo_.usedLDSSize_ = - akc->workgroup_group_segment_byte_size; - workGroupInfo_.usedSGPRs_ = akc->wavefront_sgpr_count; - workGroupInfo_.usedStackSize_ = 0; - workGroupInfo_.usedVGPRs_ = akc->workitem_vgpr_count; - - if (!prog().isNull()) { - workGroupInfo_.availableSGPRs_ = dev().gslCtx()->getNumSGPRsAvailable(); - workGroupInfo_.availableVGPRs_ = dev().gslCtx()->getNumVGPRsAvailable(); - workGroupInfo_.preferredSizeMultiple_ = dev().getAttribs().wavefrontSize; - workGroupInfo_.wavefrontPerSIMD_ = dev().getAttribs().wavefrontSize; - } else { - workGroupInfo_.availableSGPRs_ = 104; - workGroupInfo_.availableVGPRs_ = 256; - workGroupInfo_.preferredSizeMultiple_ = workGroupInfo_.wavefrontPerSIMD_ = 64; - } - return true; -} -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpusettings.cpp b/projects/clr/rocclr/device/gpu/gpusettings.cpp deleted file mode 100644 index 5251a576d2..0000000000 --- a/projects/clr/rocclr/device/gpu/gpusettings.cpp +++ /dev/null @@ -1,476 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "top.hpp" -#include "os/os.hpp" -#include "device/device.hpp" -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpusettings.hpp" - -#include - -#if defined(_WIN32) -#include "VersionHelpers.h" -#endif - -namespace gpu { - -/*! \brief information for adjusting maximum workload time - * - * This structure contains the time and OS minor version for max workload time - * adjustment for Windows 7 or 8. - */ -struct ModifyMaxWorkload { - uint32_t time; //!< max work load time (10x ms) - uint32_t minorVersion; //!< OS minor version -#if defined(_WIN32) - BYTE comparisonOps; //!< Comparison option -#endif -}; - - -Settings::Settings() { - // Initialize the GPU device default settings - oclVersion_ = OpenCL12; - debugFlags_ = 0; - syncObject_ = GPU_USE_SYNC_OBJECTS; - remoteAlloc_ = REMOTE_ALLOC; - - stagedXferRead_ = true; - stagedXferWrite_ = true; - stagedXferSize_ = GPU_STAGING_BUFFER_SIZE * Ki; - - // We will enable staged read/write if we use local memory - disablePersistent_ = false; - - maxRenames_ = 16; - maxRenameSize_ = 4 * Mi; - - imageSupport_ = false; - hwLDSSize_ = 0; - - // Set this to true when we drop the flag - doublePrecision_ = ::CL_KHR_FP64; - - // Fill workgroup info size - maxWorkGroupSize_ = 1024; - preferredWorkGroupSize_ = 256; - - hostMemDirectAccess_ = HostMemDisable; - - libSelector_ = amd::LibraryUndefined; - - // Enable workload split by default (for 24 bit arithmetic or timeout) - workloadSplitSize_ = 1 << GPU_WORKLOAD_SPLIT; - - // By default use host blit - blitEngine_ = BlitEngineHost; - const static size_t MaxPinnedXferSize = 32; - pinnedXferSize_ = std::min(GPU_PINNED_XFER_SIZE, MaxPinnedXferSize) * Mi; - pinnedMinXferSize_ = std::min(GPU_PINNED_MIN_XFER_SIZE * Ki, pinnedXferSize_); - - // Disable FP_FAST_FMA defines by default - reportFMAF_ = false; - reportFMA_ = false; - - // GPU device by default - apuSystem_ = false; - - // Disable 64 bit pointers support by default - use64BitPtr_ = false; - - // Max alloc size is 16GB - maxAllocSize_ = 16 * static_cast(Gi); - - // Disable memory dependency tracking by default - numMemDependencies_ = 0; - - // By default cache isn't present - cacheLineSize_ = 0; - cacheSize_ = 0; - - // Initialize transfer buffer size to 1MB by default - xferBufSize_ = 1024 * Ki; - - // Use image DMA if requested - imageDMA_ = GPU_IMAGE_DMA; - - // Disable ASIC specific features by default - ciPlus_ = false; - viPlus_ = false; - aiPlus_ = false; - - // Number of compute rings. - numComputeRings_ = 0; - - minWorkloadTime_ = 100; // 0.1 ms - maxWorkloadTime_ = 500000; // 500 ms - - // Controls tiled images in persistent - //!@note IOL for Linux doesn't setup tiling aperture in CMM/QS - linearPersistentImage_ = false; - - useSingleScratch_ = GPU_USE_SINGLE_SCRATCH; - - // SDMA profiling is disabled by default - sdmaProfiling_ = false; - - // Device enqueuing settings - numDeviceEvents_ = 1024; - numWaitEvents_ = 8; - - // Don't support platform atomics by default. - svmAtomics_ = false; - - // Use host queue for device enqueuing by default - useDeviceQueue_ = GPU_USE_DEVICE_QUEUE; - - // Don't support Denormals for single precision by default - singleFpDenorm_ = false; -} - -bool Settings::create(const CALdeviceattribs& calAttr, bool reportAsOCL12Device, - bool smallMemSystem) { - CALuint target = calAttr.target; - uint32_t osVer = 0x0; - - // Disable thread trace by default for all devices - threadTraceEnable_ = false; - - if (calAttr.doublePrecision) { - // Report FP_FAST_FMA define if double precision HW - reportFMA_ = true; - // FMA is 1/4 speed on Pitcairn, Cape Verde, Devastator and Scrapper - // Bonaire, Kalindi, Spectre and Spooky so disable - // FP_FMA_FMAF for those parts in switch below - reportFMAF_ = true; - } - - // Update GPU specific settings and info structure if we have any - ModifyMaxWorkload modifyMaxWorkload = {0}; - - switch (target) { - case CAL_TARGET_RAVEN: - case CAL_TARGET_RAVEN2: - case CAL_TARGET_RENOIR: - // APU systems for AI - apuSystem_ = true; - case CAL_TARGET_GREENLAND: - case CAL_TARGET_VEGA12: - case CAL_TARGET_VEGA20: - // TODO: specific codes for AI - aiPlus_ = true; - // Fall through to VI ... - case CAL_TARGET_STONEY: - if (!aiPlus_) { - // Fix BSOD/TDR issues observed on Stoney Win7/8.1/10 - minWorkloadTime_ = 1000; - modifyMaxWorkload.time = 1000; // Decided by experiment - modifyMaxWorkload.minorVersion = 1; // Win 7 -#if defined(_WIN32) - modifyMaxWorkload.comparisonOps = VER_EQUAL; // Limit to Win 7 only -#endif - } - case CAL_TARGET_CARRIZO: - if (!aiPlus_) { - // APU systems for VI - apuSystem_ = true; - } - case CAL_TARGET_ICELAND: - case CAL_TARGET_TONGA: - case CAL_TARGET_FIJI: - case CAL_TARGET_ELLESMERE: - case CAL_TARGET_BAFFIN: - case CAL_TARGET_LEXA: - case CAL_TARGET_POLARIS22: - // Disable tiling aperture on VI+ - linearPersistentImage_ = true; - // Keep this false even though we have support - // singleFpDenorm_ = true; - viPlus_ = true; - enableExtension(ClKhrFp16); - // Fall through to CI ... - case CAL_TARGET_KALINDI: - case CAL_TARGET_SPECTRE: - case CAL_TARGET_SPOOKY: - case CAL_TARGET_GODAVARI: - if (!viPlus_) { - // APU systems for CI - apuSystem_ = true; - // Fix BSOD/TDR issues observed on Kaveri Win7 (EPR#416903) - modifyMaxWorkload.time = 250000; // 250ms - modifyMaxWorkload.minorVersion = 1; // Win 7 -#if defined(_WIN32) - modifyMaxWorkload.comparisonOps = VER_EQUAL; // limit to Win 7 -#endif - } - // Fall through ... - case CAL_TARGET_BONAIRE: - case CAL_TARGET_HAWAII: - ciPlus_ = true; - sdmaProfiling_ = true; - threadTraceEnable_ = AMD_THREAD_TRACE_ENABLE; - reportFMAF_ = false; - if (target == CAL_TARGET_HAWAII) { - reportFMAF_ = true; - } - // Cache line size is 64 bytes - cacheLineSize_ = 64; - // L1 cache size is 16KB - cacheSize_ = 16 * Ki; - - if (ciPlus_) { - libSelector_ = amd::GPU_Library_CI; - if (LP64_SWITCH(false, true)) { - oclVersion_ = !reportAsOCL12Device && calAttr.isOpenCL200Device - ? XCONCAT(OpenCL, XCONCAT(OPENCL_MAJOR, OPENCL_MINOR)) - : OpenCL12; - } - if (smallMemSystem) { // force the dGPU to be 1.2 device for small memory system. - if (apuSystem_) { - return false; - } else { - oclVersion_ = OpenCL12; - } - } - if (GPU_FORCE_OCL20_32BIT) { - force32BitOcl20_ = true; - oclVersion_ = !reportAsOCL12Device && calAttr.isOpenCL200Device - ? XCONCAT(OpenCL, XCONCAT(OPENCL_MAJOR, OPENCL_MINOR)) - : OpenCL12; - } - if (OPENCL_VERSION < 200) { - oclVersion_ = OpenCL12; - } - numComputeRings_ = 8; - } else { - numComputeRings_ = 2; - libSelector_ = amd::GPU_Library_SI; - } - - // Cap at OpenCL20 for now - if (oclVersion_ > OpenCL20) oclVersion_ = OpenCL20; - - // This needs to be cleaned once 64bit addressing is stable - if (oclVersion_ < OpenCL20) { - use64BitPtr_ = flagIsDefault(GPU_FORCE_64BIT_PTR) - ? LP64_SWITCH(false, true) - : GPU_FORCE_64BIT_PTR; - } else { - if (GPU_FORCE_64BIT_PTR || LP64_SWITCH(false, true)) { - use64BitPtr_ = true; - } - } - - if (oclVersion_ >= OpenCL20) { - supportDepthsRGB_ = true; - } - if (use64BitPtr_) { - if (GPU_ENABLE_LARGE_ALLOCATION && (viPlus_ || (oclVersion_ == OpenCL20))) { - maxAllocSize_ = 64ULL * Gi; - } else { - maxAllocSize_ = 4048 * Mi; - } - } else { - maxAllocSize_ = 3ULL * Gi; - } - - supportRA_ = false; - numMemDependencies_ = GPU_NUM_MEM_DEPENDENCY; - - enableExtension(ClKhrInt64BaseAtomics); - enableExtension(ClKhrInt64ExtendedAtomics); - enableExtension(ClKhrImage2dFromBuffer); - break; - default: - assert(0 && "Unknown ASIC type!"); - return false; - } - -#if defined(_WIN32) - if (modifyMaxWorkload.time > 0) { - OSVERSIONINFOEX versionInfo = {0}; - versionInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX); - versionInfo.dwMajorVersion = 6; - versionInfo.dwMinorVersion = modifyMaxWorkload.minorVersion; - - DWORDLONG conditionMask = 0; - VER_SET_CONDITION(conditionMask, VER_MAJORVERSION, modifyMaxWorkload.comparisonOps); - VER_SET_CONDITION(conditionMask, VER_MINORVERSION, modifyMaxWorkload.comparisonOps); - if (VerifyVersionInfo(&versionInfo, VER_MAJORVERSION | VER_MINORVERSION, conditionMask)) { - maxWorkloadTime_ = modifyMaxWorkload.time; - } - } - enableExtension(ClAMDLiquidFlash); -#endif // defined(_WIN32) - - // Enable atomics support - enableExtension(ClKhrGlobalInt32BaseAtomics); - enableExtension(ClKhrGlobalInt32ExtendedAtomics); - enableExtension(ClKhrLocalInt32BaseAtomics); - enableExtension(ClKhrLocalInt32ExtendedAtomics); - enableExtension(ClKhrByteAddressableStore); - enableExtension(ClKhrGlSharing); - enableExtension(ClKhrGlEvent); - enableExtension(ClAmdMediaOps); - enableExtension(ClAmdMediaOps2); - enableExtension(ClAmdPopcnt); - enableExtension(ClKhr3DImageWrites); - enableExtension(ClAmdVec3); - enableExtension(ClAmdPrintf); - // Enable some platform extensions - enableExtension(ClAmdDeviceAttributeQuery); - enableExtension(ClKhrSpir); - - hwLDSSize_ = 32 * Ki; - - imageSupport_ = true; - - // Use kernels for blit if appropriate - blitEngine_ = BlitEngineKernel; - - hostMemDirectAccess_ |= HostMemBuffer; - // HW doesn't support untiled image writes - // hostMemDirectAccess_ |= HostMemImage; - - // Make sure device actually supports double precision - doublePrecision_ = (calAttr.doublePrecision) ? doublePrecision_ : false; - if (doublePrecision_) { - // Enable KHR double precision extension - enableExtension(ClKhrFp64); - } - - if (calAttr.doublePrecision) { - // Enable AMD double precision extension - doublePrecision_ = true; - enableExtension(ClAmdFp64); - } - - if (calAttr.totalSDIHeap > 0) { - // Enable bus addressable memory extension - enableExtension(ClAMDBusAddressableMemory); - } - - if (calAttr.longIdleDetect) { - // KMD is unable to detect if we map the visible memory for CPU access, so - // accessing persistent staged buffer may fail if LongIdleDetct is enabled. - disablePersistent_ = true; - } - - svmFineGrainSystem_ = calAttr.isSVMFineGrainSystem; - - svmAtomics_ = (calAttr.svmAtomics || calAttr.isSVMFineGrainSystem) ? true : false; - -#if defined(_WIN32) - enableExtension(ClKhrD3d9Sharing); - enableExtension(ClKhrD3d10Sharing); - enableExtension(ClKhrD3d11Sharing); -#endif // _WIN32 - - // Enable some OpenCL 2.0 extensions - if (oclVersion_ >= OpenCL20) { - enableExtension(ClKhrGLDepthImages); - enableExtension(ClKhrSubGroups); - enableExtension(ClKhrDepthImages); - - if (GPU_MIPMAP) { - enableExtension(ClKhrMipMapImage); - enableExtension(ClKhrMipMapImageWrites); - } - - // Enable HW debug - if (GPU_ENABLE_HW_DEBUG) { - enableHwDebug_ = true; - } - -#if defined(_WIN32) - enableExtension(ClAmdPlanarYuv); -#endif - } - - if (apuSystem_ && ((calAttr.totalVisibleHeap + calAttr.totalInvisibleHeap) < 150)) { - remoteAlloc_ = true; - } - -// Save resource cache size -#ifdef ATI_OS_LINUX - // Due to EPR#406216, set the default value for Linux for now - resourceCacheSize_ = GPU_RESOURCE_CACHE_SIZE * Mi; -#else - if (remoteAlloc_) { - resourceCacheSize_ = - std::max((calAttr.uncachedRemoteRAM / 8) * Mi, GPU_RESOURCE_CACHE_SIZE * Mi); - } else { - resourceCacheSize_ = std::max((calAttr.localRAM / 8) * Mi, GPU_RESOURCE_CACHE_SIZE * Mi); - } -#endif - - // Override current device settings - override(); - - return true; -} - -void Settings::override() { - // Limit reported workgroup size - if (GPU_MAX_WORKGROUP_SIZE != 0) { - preferredWorkGroupSize_ = GPU_MAX_WORKGROUP_SIZE; - } - - // Override blit engine type - if (GPU_BLIT_ENGINE_TYPE != BlitEngineDefault) { - blitEngine_ = GPU_BLIT_ENGINE_TYPE; - } - - if (!flagIsDefault(DEBUG_GPU_FLAGS)) { - debugFlags_ = DEBUG_GPU_FLAGS; - } - - if (!flagIsDefault(GPU_XFER_BUFFER_SIZE)) { - xferBufSize_ = GPU_XFER_BUFFER_SIZE * Ki; - } - - if (!flagIsDefault(GPU_USE_SYNC_OBJECTS)) { - syncObject_ = GPU_USE_SYNC_OBJECTS; - } - - if (!flagIsDefault(GPU_NUM_COMPUTE_RINGS)) { - numComputeRings_ = GPU_NUM_COMPUTE_RINGS; - } - - if (!flagIsDefault(GPU_RESOURCE_CACHE_SIZE)) { - resourceCacheSize_ = GPU_RESOURCE_CACHE_SIZE * Mi; - } - - if (!flagIsDefault(AMD_GPU_FORCE_SINGLE_FP_DENORM)) { - switch (AMD_GPU_FORCE_SINGLE_FP_DENORM) { - case 0: - singleFpDenorm_ = false; - break; - case 1: - singleFpDenorm_ = true; - break; - default: - break; - } - } -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpusettings.hpp b/projects/clr/rocclr/device/gpu/gpusettings.hpp deleted file mode 100644 index b51f8aa2b2..0000000000 --- a/projects/clr/rocclr/device/gpu/gpusettings.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* Copyright (c) 2010 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUSETTINGS_HPP_ -#define GPUSETTINGS_HPP_ - -#include "top.hpp" -#include "library.hpp" - -/*! \addtogroup GPU GPU Resource Implementation - * @{ - */ - -//! GPU Device Implementation -namespace gpu { - -//! Device settings -class Settings : public device::Settings { - public: - //! Debug GPU flags - enum DebugGpuFlags { - CheckForILSource = 0x00000001, - StubCLPrograms = 0x00000002, //!< Enables OpenCL programs stubbing - LockGlobalMemory = 0x00000004, - }; - - enum BlitEngineType { - BlitEngineDefault = 0x00000000, - BlitEngineHost = 0x00000001, - BlitEngineCAL = 0x00000002, - BlitEngineKernel = 0x00000003, - }; - - enum HostMemFlags { - HostMemDisable = 0x00000000, - HostMemBuffer = 0x00000001, - HostMemImage = 0x00000002, - }; - - union { - struct { - uint remoteAlloc_ : 1; //!< Allocate remote memory for the heap - uint stagedXferRead_ : 1; //!< Uses a staged buffer read - uint stagedXferWrite_ : 1; //!< Uses a staged buffer write - uint disablePersistent_ : 1; //!< Disables using persistent memory for staging - uint imageSupport_ : 1; //!< Report images support - uint doublePrecision_ : 1; //!< Enables double precision support - uint use64BitPtr_ : 1; //!< Use 64bit pointers on GPU - uint force32BitOcl20_ : 1; //!< Force 32bit apps to take CLANG/HSAIL path on GPU - uint imageDMA_ : 1; //!< Enable direct image DMA transfers - uint syncObject_ : 1; //!< Enable syncobject - uint ciPlus_ : 1; //!< CI and post CI features - uint viPlus_ : 1; //!< VI and post VI features - uint aiPlus_ : 1; //!< AI and post AI features - uint threadTraceEnable_ : 1; //!< Thread trace enable - uint linearPersistentImage_ : 1; //!< Allocates linear images in persistent - uint useSingleScratch_ : 1; //!< Allocates single scratch per device - uint sdmaProfiling_ : 1; //!< Enables SDMA profiling - uint svmAtomics_ : 1; //!< SVM device atomics - uint svmFineGrainSystem_ : 1; //!< SVM fine grain system support - uint useDeviceQueue_ : 1; //!< Submit to separate device queue - uint reserved_ : 11; - }; - uint value_; - }; - - uint oclVersion_; //!< Reported OpenCL version support - uint debugFlags_; //!< Debug GPU flags - size_t stagedXferSize_; //!< Staged buffer size - uint maxRenames_; //!< Maximum number of possible renames - uint maxRenameSize_; //!< Maximum size for all renames - uint hwLDSSize_; //!< HW local data store size - uint maxWorkGroupSize_; //!< Requested workgroup size for this device - uint preferredWorkGroupSize_; //!< Requested preferred workgroup size for this device - uint hostMemDirectAccess_; //!< Enables direct access to the host memory - amd::LibrarySelector libSelector_; //!< Select linking libraries for compiler - uint workloadSplitSize_; //!< Workload split size - uint minWorkloadTime_; //!< Minimal workload time in 0.1 ms - uint maxWorkloadTime_; //!< Maximum workload time in 0.1 ms - uint blitEngine_; //!< Blit engine type - size_t pinnedXferSize_; //!< Pinned buffer size for transfer - size_t pinnedMinXferSize_; //!< Minimal buffer size for pinned transfer - size_t resourceCacheSize_; //!< Resource cache size in MB - uint64_t maxAllocSize_; //!< Maximum single allocation size - size_t numMemDependencies_; //!< The array size for memory dependencies tracking - uint cacheLineSize_; //!< Cache line size in bytes - uint cacheSize_; //!< L1 cache size in bytes - size_t xferBufSize_; //!< Transfer buffer size for image copy optimization - uint numComputeRings_; //!< 0 - disabled, 1 , 2,.. - the number of compute rings - uint numDeviceEvents_; //!< The number of device events - uint numWaitEvents_; //!< The number of wait events for device enqueue - - - //! Default constructor - Settings(); - - //! Creates settings - bool create(const CALdeviceattribs& calAttr //!< CAL attributes structure - , - bool reportAsOCL12Device = false //!< Report As OpenCL1.2 Device - , - bool smallMemSystem = false //!< report the sys memory is small - ); - - private: - //! Disable copy constructor - Settings(const Settings&); - - //! Disable assignment - Settings& operator=(const Settings&); - - //! Overrides current settings based on registry/environment - void override(); -}; - -/*@}*/} // namespace gpu - -#endif /*GPUSETTINGS_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gputhreadtrace.cpp b/projects/clr/rocclr/device/gpu/gputhreadtrace.cpp deleted file mode 100644 index 95faa038e3..0000000000 --- a/projects/clr/rocclr/device/gpu/gputhreadtrace.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "device/gpu/gputhreadtrace.hpp" -#include "device/gpu/gpuvirtual.hpp" - -namespace gpu { - -CalThreadTraceReference::~CalThreadTraceReference() { - // The thread trace object is always associated with a particular queue, - // so we have to lock just this queue - amd::ScopedLock lock(gpu_.execution()); - - if (0 != threadTrace_) { - // gpu().cs()->destroyQuery(gslThreadTrace()); - } -} - - -ThreadTrace::~ThreadTrace() { - if (calRef_ == NULL) { - return; - } - for (uint i = 0; i < amdThreadTraceMemObjsNum_; ++i) { - threadTraceBufferObjs_[i]->attachMemObject(gpu().cs(), NULL, 0, 0, 0, i); - gpu().cs()->destroyShaderTraceBuffer(threadTraceBufferObjs_[i]); - } - - // Release the thread trace reference object - // calRef_->release(); -} - -bool ThreadTrace::create(CalThreadTraceReference* calRef) { - assert(&gpu() == &calRef->gpu()); - - calRef_ = calRef; - threadTrace_ = calRef->gslThreadTrace(); - - return true; -} - -bool ThreadTrace::info(uint infoType, uint* info, uint infoSize) const { - switch (infoType) { - case CL_THREAD_TRACE_BUFFERS_SIZE: { - if (infoSize < amdThreadTraceMemObjsNum_) { - LogError("The amount of buffers should be equal to the amount of Shader Engines"); - return false; - } else { - gslThreadTrace()->GetResultAll(gpu().cs(), info); - } - break; - } - default: - LogError("Wrong ThreadTrace::getInfo parameter"); - return false; - } - return true; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gputhreadtrace.hpp b/projects/clr/rocclr/device/gpu/gputhreadtrace.hpp deleted file mode 100644 index 4b37895603..0000000000 --- a/projects/clr/rocclr/device/gpu/gputhreadtrace.hpp +++ /dev/null @@ -1,148 +0,0 @@ -/* Copyright (c) 2009 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPU_THREAD_TRACE_HPP_ -#define GPU_THREAD_TRACE_HPP_ - -#include "top.hpp" -#include "device/device.hpp" -#include "device/gpu/gpudevice.hpp" -#include "shadertracebuffer/ShaderTraceBufferObject.h" - -#include -namespace gpu { - -class VirtualGPU; - -class CalThreadTraceReference : public amd::ReferenceCountedObject { - public: - //! Default constructor - CalThreadTraceReference(VirtualGPU& gpu, //!< Virtual GPU device object - gslQueryObject gslThreadTrace) //!< GSL query thread trace object - : gpu_(gpu), - threadTrace_(gslThreadTrace) {} - - //! Get GSL thread race object - gslQueryObject gslThreadTrace() const { return threadTrace_; } - - //! Returns the virtual GPU device - const VirtualGPU& gpu() const { return gpu_; } - - protected: - //! Default destructor - ~CalThreadTraceReference(); - - private: - //! Disable copy constructor - CalThreadTraceReference(const CalThreadTraceReference&); - - //! Disable operator= - CalThreadTraceReference& operator=(const CalThreadTraceReference&); - - VirtualGPU& gpu_; //!< The virtual GPU device object - gslQueryObject threadTrace_; //!< GSL thread trace query object -}; - -//! ThreadTrace implementation on GPU -class ThreadTrace : public device::ThreadTrace { - public: - //! Destructor for the GPU ThreadTrace object - virtual ~ThreadTrace(); - - //! Creates the current object - bool create(CalThreadTraceReference* calRef //!< Reference ThreadTrace - ); - - //! Returns the GPU device, associated with the current object - const Device& dev() const { return gpuDevice_; } - - //! Returns the virtual GPU device - const VirtualGPU& gpu() const { return gpu_; } - - //! Constructor for the GPU ThreadTrace object - ThreadTrace(Device& device, //!< A GPU device object - VirtualGPU& gpu, //!< Virtual GPU device object - uint amdThreadTraceMemObjsNum) - : gpuDevice_(device), - gpu_(gpu), - calRef_(NULL), - index_(0), - amdThreadTraceMemObjsNum_(amdThreadTraceMemObjsNum) { - threadTraceBufferObjs_ = new gslShaderTraceBufferObject[amdThreadTraceMemObjsNum]; - for (uint i = 0; i < amdThreadTraceMemObjsNum; ++i) { - threadTraceBufferObjs_[i] = gpu.cs()->createShaderTraceBuffer(); - } - } - - //! Returns the specific information about the thread trace object - bool info(uint infoType, //!< The type of returned information - uint* info, //!< The returned information - uint infoSize //!< The size of returned information - ) const; - - //! Set the ThreadTrace memory buffer size - void setMemBufferSizeTT(uint memBufferSizeTT) { memBufferSizeTT_ = memBufferSizeTT; } - - //! Set isNewBufferBinded_ to true/false if new buffer was binded/unbinded respectively - void setNewBufferBinded(bool isNewBufferBinded) { isNewBufferBinded_ = isNewBufferBinded; } - - //! Attach gslMemObject to the TreadTrace buffer - void attachMemToThreadTraceBuffer(); - - void setMemObj(size_t memObjSize, std::vector memObj) { - memObj_ = memObj; - memBufferSizeTT_ = memObjSize; - } - //! Get GSL thread trace object - gslQueryObject gslThreadTrace() const { return threadTrace_; } - - //! Get GSL Thread Trace Buffer objects - gslShaderTraceBufferObject* getThreadTraceBufferObjects() { return threadTraceBufferObjs_; } - - private: - //! Disable default copy constructor - ThreadTrace(const ThreadTrace&); - - //! Disable default operator= - ThreadTrace& operator=(const ThreadTrace&); - - //! Retrieve gslMemoryObject - gslMemObject getCurrentGslMemObject(amd::Memory*); - - const Device& gpuDevice_; //!< The backend device - - VirtualGPU& gpu_; //!< The virtual GPU device object - - CalThreadTraceReference* calRef_; //!< Reference ThreadTrace - gslShaderTraceBufferObject* - threadTraceBufferObjs_; //!< The buffer object for Thread Trace recording - uint index_; //!< ThreadTrace index in the CAL container - uint memBufferSizeTT_; //!< ThreadTrace memory buffer size - std::vector memObj_; //!< ThreadTrace memory object - gslQueryObject threadTrace_; //!< GSL thread trace query object - uint amdThreadTraceMemObjsNum_; //!< ThreadTrace memory object`s number (should be equal to the - //!SE number) - bool isNewBufferBinded_; //!< The indicator if new buffer was binded to the ThreadTrace object - bool isBufferOnSubmit_; //!< The indicator if "new buffer on submit" mode is used -}; - -} // namespace gpu - -#endif // GPU_THREAD_TRACE_HPP_ diff --git a/projects/clr/rocclr/device/gpu/gputimestamp.cpp b/projects/clr/rocclr/device/gpu/gputimestamp.cpp deleted file mode 100644 index a21c655d5c..0000000000 --- a/projects/clr/rocclr/device/gpu/gputimestamp.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "os/os.hpp" -#include "platform/perfctr.hpp" -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gputimestamp.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "device/gpu/gpucounters.hpp" - -namespace gpu { - -TimeStamp::TimeStamp(const VirtualGPU& gpu, gslMemObject gslMem, uint memOffset, address cpuAddr) - : gpu_(gpu), gslMem_(gslMem), memOffset_(memOffset) { - values_ = reinterpret_cast(cpuAddr + memOffset); -} - -TimeStamp::~TimeStamp() {} - -void TimeStamp::begin(bool sdma) { - if (!flags_.beginIssued_) { - gpu().rs()->writeTimer(gpu().cs(), sdma, gslMem_, - memOffset_ + CommandStartTime * sizeof(uint64_t)); - flags_.beginIssued_ = true; - } -} - -void TimeStamp::end(bool sdma) { - CondLog(!flags_.beginIssued_, "We didn't issue a begin operation!"); - gpu().rs()->writeTimer(gpu().cs(), sdma, gslMem_, memOffset_ + CommandEndTime * sizeof(uint64_t)); - flags_.endIssued_ = true; - flags_.sdma_ = sdma; -} - -inline void SetValue(uint64_t* time, uint64_t val, double nanos) { - *time = static_cast(static_cast(val) * nanos); -} - -void TimeStamp::value(uint64_t* startTime, uint64_t* endTime) { - CondLog(!flags_.endIssued_, "We didn't send the counter end operation!"); - const double NanoSecondsPerTick = gpu_.dev().getAttribs().nanoSecondsPerTick; - - SetValue(startTime, values_[CommandStartTime], NanoSecondsPerTick); - SetValue(endTime, values_[CommandEndTime], NanoSecondsPerTick); -} - -TimeStampCache::~TimeStampCache() { - // Release all time stamp objects from the cache - for (uint i = 0; i < freedTS_.size(); ++i) { - delete freedTS_[i]; - } - freedTS_.clear(); - - // Release all memory objects - for (uint i = 0; i < tsBuf_.size(); ++i) { - tsBuf_[i]->unmap(&gpu_); - delete tsBuf_[i]; - } - tsBuf_.clear(); -} - -TimeStamp* TimeStampCache::allocTimeStamp() { - TimeStamp* ts = NULL; - if (0 != freedTS_.size()) { - ts = freedTS_.back(); - freedTS_.pop_back(); - } - - if (NULL == ts) { - if ((tsBufCpu_ == NULL) || ((tsOffset_ + TimerSlotSize) > TimerBufSize)) { - Memory* buf = new Memory(gpu_.dev(), TimerBufSize); - if (buf == NULL || !buf->create(Resource::Remote)) { - return NULL; - } - tsBufCpu_ = reinterpret_cast
(buf->map(&gpu_)); - memset(tsBufCpu_, 0, TimerBufSize); - tsOffset_ = 0; - tsBuf_.push_back(buf); - } - // Allocate a TimeStamp object - ts = new TimeStamp(gpu_, tsBuf_[(tsBuf_.size() - 1)]->gslResource(), tsOffset_, tsBufCpu_); - // Create a timestamp - if (ts == NULL) { - return NULL; - } - tsOffset_ += TimerSlotSize; - } - - // Set this timestamp into DRM profile mode if it was requested - ts->clearStates(); - - return ts; -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gputimestamp.hpp b/projects/clr/rocclr/device/gpu/gputimestamp.hpp deleted file mode 100644 index 007c9d59c8..0000000000 --- a/projects/clr/rocclr/device/gpu/gputimestamp.hpp +++ /dev/null @@ -1,137 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUTIMESTAMP_HPP_ -#define GPUTIMESTAMP_HPP_ - -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpuresource.hpp" - -/*! \addtogroup GPU GPU Resource Implementation - * @{ - */ - -//! GPU Device Implementation -namespace gpu { - -class Device; -class VirtualGPU; -class Memory; - -class TimeStamp : public amd::HeapObject { - public: - //! Enums for the timestamp information - //! \note *4 is the limitaiton of SDMA HW - //! (address has to be aligned by 256 bit) - enum TimeStampValue { CommandStartTime = 0, CommandEndTime = 4, CommandTotal = 8 }; - - //! The TimeStamp object flags - union Flags { - struct { - uint32_t beginIssued_ : 1; - uint32_t endIssued_ : 1; - uint32_t sdma_ : 1; - }; - uint32_t value_; - Flags() : value_(0) {} - }; - - //! Default constructor - TimeStamp(const VirtualGPU& gpu, //!< Virtual GPU - gslMemObject gslMem, //!< Buffer with the timer values - uint memOffset, //!< Offset in the buffer for the current TS - address cpuAddr //!< CPU pointer for the values in memory - ); - - //! Default destructor - ~TimeStamp(); - - //! Starts the timestamp - void begin(bool sdma = false); - - //! Ends the timestamp - void end(bool sdma = false); - - //! Returns the timestamp result in nano seconds - void value(uint64_t* startTime, uint64_t* endTime); - - //! Clear all TimeStamp states - void clearStates() { - flags_.value_ = 0; - values_[CommandStartTime] = 0; - values_[CommandEndTime] = 0; - } - - //! Timer commands were submitted to HW - bool isValid() const { return (flags_.endIssued_) ? true : false; } - - private: - //! Disable copy constructor - TimeStamp(const TimeStamp&); - - //! Disable operator= - TimeStamp& operator=(const TimeStamp&); - - //! Returns the GPU device object - const VirtualGPU& gpu() const { return gpu_; } - - const VirtualGPU& gpu_; //!< Virtual GPU - Flags flags_; //!< The time stamp state - gslMemObject gslMem_; //!< Buffer with the timer values - uint memOffset_; //!< Offset in the buffer for the current timer - volatile uint64_t* values_; //!< CPU pointer to the timer values -}; - -class TimeStampCache : public amd::HeapObject { - public: - //! Default constructor - TimeStampCache(VirtualGPU& gpu //!< Virtual GPU object - ) - : gpu_(gpu), tsBufCpu_(NULL), tsOffset_(0) {} - - //! Default destructor - ~TimeStampCache(); - - //! Gets a time stamp object. It will find a freed object or allocate a new one - TimeStamp* allocTimeStamp(); - - //! Frees a time stamp object - void freeTimeStamp(TimeStamp* ts) { freedTS_.push_back(ts); } - - private: - static constexpr uint TimerSlotSize = TimeStamp::CommandTotal * sizeof(uint64_t); - static constexpr uint TimerBufSize = TimerSlotSize * 4096; - - //! Disable copy constructor - TimeStampCache(const TimeStampCache&); - - //! Disable operator= - TimeStampCache& operator=(const TimeStampCache&); - - std::vector freedTS_; //!< Array of freed time stamp objects - VirtualGPU& gpu_; //!< Virtual GPU - std::vector tsBuf_; //!< Array of memory objects with the timer value - address tsBufCpu_; //!< CPU pointer for current TS memory - uint tsOffset_; //!< Active offset in the current mem object -}; - -/*@}*/} // namespace gpu - -#endif /*GPUTIMESTAMP_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gputrap.hpp b/projects/clr/rocclr/device/gpu/gputrap.hpp deleted file mode 100644 index 9a7517e9ea..0000000000 --- a/projects/clr/rocclr/device/gpu/gputrap.hpp +++ /dev/null @@ -1,139 +0,0 @@ -/******************************************************************************* - * The source of the runtime trap handler, "runtimetraphandler.sp3". - * The binary is created by the SP3 tool with the following command: - * - * sp3.exe runtimetraphandler.sp3 -hex runtimeTrapCode.hex - * - ******************************************************************************* - -shader main - asic(TAHITI) // for SI/CI or asic(VI) for VI - type(CS) - - // clear wave exception state - v_clrexcp - s_waitcnt 0 - //========================================================================== - // Handle the workaround for HW bug that causes the incorrect TMA value. - // Retrieve the TMA values, which are stored at TBA buffer at location - // 256 (0x100). - - // Construct the memory descriptor with TBA as the start address - // we are using the registers ttmp[8:11] for that. - s_mov_b32 ttmp8, tba_lo - s_and_b32 ttmp9, tba_hi, 0xffff - - // 0x100=256 bytes, which is the size of the buffer to - // store all the level 2 trap handler info - s_or_b32 ttmp9, ttmp9, 0x01000000 - s_mov_b32 ttmp10, 0x00002000 - s_mov_b32 ttmp11, 0x00024fac - - // TMA is stored 256 (0x100) bytes before the TBA value - s_sub_u32 ttmp8, ttmp8, 0x100 - - // Backup the s0 since ttmp registers cannot be target of - // buffer read instruction - s_mov_b32 ttmp7, s0 - s_buffer_load_dword s0, ttmp8, 0x0 // VI: offset=0x0 (bytes) - s_waitcnt 0 - s_mov_b32 tma_lo, s0 - s_buffer_load_dword s0, ttmp8, 0x1 // VI: offset=0x4 (bytes) - s_waitcnt 0 - s_mov_b32 tma_hi, s0 - s_mov_b32 s0, ttmp7 - - //=================================================== - // setup the mmeory descriptor for TMA - s_mov_b32 ttmp6, 0x18 - s_add_u32 ttmp8, tma_lo, ttmp6 - s_and_b32 ttmp9, tma_hi, 0xffff - //0x68=104 bytes, which is the size of the buffer to - //store all the level2 trap handler info - s_or_b32 ttmp9, ttmp9, 0x00680000 - s_mov_b32 ttmp10, 0x00002000 - s_mov_b32 ttmp11, 0x00024fac - - //=================================================== - // backup the TMA values to be restored later - // level-one TMA saved in the ttmp6,ttmp7 - s_mov_b32 ttmp6, tma_lo - s_mov_b32 ttmp7, tma_hi - - //=================================================== - // setup the TMA for the level-two trap handler - // level-two TMA saved in tma_hi, tma_lo - s_mov_b32 ttmp3, s0 - s_buffer_load_dword s0, ttmp8, 0x2 // VI: offset=0x8 (bytes) - s_waitcnt 0x0000 - s_mov_b32 tma_lo, s0 - - s_buffer_load_dword s0, ttmp8, 0x3 // VI: offset=0xc (bytes) - s_waitcnt 0x0000 - s_mov_b32 tma_hi, s0 - - //=================================================== - // setup the TBA for the level-two trap handler - // level-two TBA saved in ttmp9, ttmp8 - s_buffer_load_dword s0, ttmp8, 0x0 // VI: offset=0x0 (bytes) - s_waitcnt 0x0000 - s_mov_b32 ttmp2, s0 - - s_buffer_load_dword s0, ttmp8, 0x1 // VI: offset=0x4 (bytes) - s_waitcnt 0x0000 - - //swap the values of s0 and ttmp3 without using other registers - s_xor_b32 ttmp3, s0, ttmp3 - s_xor_b32 s0, s0, ttmp3 - s_xor_b32 ttmp3, s0, ttmp3 - - //store the debug trap handler start address in ttmp8,9 - s_mov_b32 ttmp8, ttmp2 - s_mov_b32 ttmp9, ttmp3 - - //=================================================== - // get the pc value to resume execution - s_getpc_b64 [ttmp2, ttmp3] - s_add_u32 ttmp2, ttmp2, 0x8 - - //=================================================== - //set the pc value to jump to the debug trap handler - s_setpc_b64 [ttmp8, ttmp9] - - //=================================================== - // restore the tamp values - s_mov_b32 tma_hi, ttmp7 - s_mov_b32 tma_lo, ttmp6 - - label_return: - //=================================================== - // return from the trap handler to the saved PC - s_and_b32 ttmp1, ttmp1, 0xffff - s_rfe_b64 [ttmp0,ttmp1] - -end - -*******************************************************************************/ - -/// shader codes with "asic(TAHITI)" instruction -static constexpr uint32_t RuntimeTrapCode[] = { - 0x7e008200, 0xbf8c0000, 0xbef8036c, 0x8779ff6d, 0x0000ffff, 0x8879ff79, 0x01000000, 0xbefa03ff, - 0x00002000, 0xbefb03ff, 0x00024fac, 0x80f8ff78, 0x00000100, 0xbef70300, 0xc2007900, 0xbf8c0000, - 0xbeee0300, 0xc2007901, 0xbf8c0000, 0xbeef0300, 0xbe800377, 0xbef60398, 0x8078766e, 0x8779ff6f, - 0x0000ffff, 0x8879ff79, 0x00680000, 0xbefa03ff, 0x00002000, 0xbefb03ff, 0x00024fac, 0xbef6036e, - 0xbef7036f, 0xbef30300, 0xc2007902, 0xbf8c0000, 0xbeee0300, 0xc2007903, 0xbf8c0000, 0xbeef0300, - 0xc2007900, 0xbf8c0000, 0xbef20300, 0xc2007901, 0xbf8c0000, 0x89737300, 0x89007300, 0x89737300, - 0xbef80372, 0xbef90373, 0xbef21f00, 0x80728872, 0xbe802078, 0xbeef0377, 0xbeee0376, 0x8771ff71, - 0x0000ffff, 0xbe802270}; - - -/// shader codes with "asic(VI)" instruction -static constexpr uint32_t RuntimeTrapCodeVi[] = { - 0x7e006a00, 0xbf8c0000, 0xbef8006c, 0x8679ff6d, 0x0000ffff, 0x8779ff79, 0x01000000, 0xbefa00ff, - 0x00002000, 0xbefb00ff, 0x00024fac, 0x80f8ff78, 0x00000100, 0xbef70000, 0xc022003c, 0x00000000, - 0xbf8c0000, 0xbeee0000, 0xc022003c, 0x00000004, 0xbf8c0000, 0xbeef0000, 0xbe800077, 0xbef60098, - 0x8078766e, 0x8679ff6f, 0x0000ffff, 0x8779ff79, 0x00680000, 0xbefa00ff, 0x00002000, 0xbefb00ff, - 0x00024fac, 0xbef6006e, 0xbef7006f, 0xbef30000, 0xc022003c, 0x00000008, 0xbf8c0000, 0xbeee0000, - 0xc022003c, 0x0000000c, 0xbf8c0000, 0xbeef0000, 0xc022003c, 0x00000000, 0xbf8c0000, 0xbef20000, - 0xc022003c, 0x00000004, 0xbf8c0000, 0x88737300, 0x88007300, 0x88737300, 0xbef80072, 0xbef90073, - 0xbef21c00, 0x80728872, 0xbe801d78, 0xbeef0077, 0xbeee0076, 0x8671ff71, 0x0000ffff, 0xbe801f70}; diff --git a/projects/clr/rocclr/device/gpu/gpuvirtual.cpp b/projects/clr/rocclr/device/gpu/gpuvirtual.cpp deleted file mode 100644 index da0cea2218..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuvirtual.cpp +++ /dev/null @@ -1,3333 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "platform/perfctr.hpp" -#include "platform/threadtrace.hpp" -#include "platform/kernel.hpp" -#include "platform/commandqueue.hpp" -#include "device/gpu/gpuconstbuf.hpp" -#include "device/gpu/gpuvirtual.hpp" -#include "device/gpu/gpukernel.hpp" -#include "device/gpu/gpuprogram.hpp" -#include "device/gpu/gpucounters.hpp" -#include "device/gpu/gputhreadtrace.hpp" -#include "device/gpu/gputimestamp.hpp" -#include "device/gpu/gpublit.hpp" -#include "device/gpu/gpudebugger.hpp" -#include "shader/ComputeProgramObject.h" -#include "hsa.h" -#include "amd_hsa_kernel_code.h" -#include "amd_hsa_queue.h" -#include -#include -#include - -#ifdef _WIN32 -#include -#include "amdocl/cl_d3d9_amd.hpp" -#include "amdocl/cl_d3d10_amd.hpp" -#include "amdocl/cl_d3d11_amd.hpp" -#endif // _WIN32 - -namespace gpu { - -bool VirtualGPU::MemoryDependency::create(size_t numMemObj) { - if (numMemObj > 0) { - // Allocate the array of memory objects for dependency tracking - memObjectsInQueue_ = new MemoryState[numMemObj]; - if (NULL == memObjectsInQueue_) { - return false; - } - memset(memObjectsInQueue_, 0, sizeof(MemoryState) * numMemObj); - maxMemObjectsInQueue_ = numMemObj; - } - - return true; -} - -void VirtualGPU::MemoryDependency::validate(VirtualGPU& gpu, const Memory* memory, bool readOnly) { - bool flushL1Cache = false; - - if (maxMemObjectsInQueue_ == 0) { - // Flush cache - gpu.flushCUCaches(); - return; - } - - uint64_t curStart = memory->hbOffset(); - uint64_t curEnd = curStart + memory->hbSize(); - - // Loop through all memory objects in the queue and find dependency - // @note don't include objects from the current kernel - for (size_t j = 0; j < endMemObjectsInQueue_; ++j) { - // Check if the queue already contains this mem object and - // GPU operations aren't readonly - uint64_t busyStart = memObjectsInQueue_[j].start_; - uint64_t busyEnd = memObjectsInQueue_[j].end_; - - // Check if the start inside the busy region - if ((((curStart >= busyStart) && (curStart < busyEnd)) || - // Check if the end inside the busy region - ((curEnd > busyStart) && (curEnd <= busyEnd)) || - // Check if the start/end cover the busy region - ((curStart <= busyStart) && (curEnd >= busyEnd))) && - // If the buys region was written or the current one is for write - (!memObjectsInQueue_[j].readOnly_ || !readOnly)) { - flushL1Cache = true; - break; - } - } - - // Did we reach the limit? - if (maxMemObjectsInQueue_ <= numMemObjectsInQueue_) { - flushL1Cache = true; - } - - if (flushL1Cache) { - // Flush cache - gpu.flushCUCaches(); - - // Clear memory dependency state - const static bool All = true; - clear(!All); - } - - // Insert current memory object into the queue always, - // since runtime calls flush before kernel execution and it has to keep - // current kernel in tracking - memObjectsInQueue_[numMemObjectsInQueue_].start_ = curStart; - memObjectsInQueue_[numMemObjectsInQueue_].end_ = curEnd; - memObjectsInQueue_[numMemObjectsInQueue_].readOnly_ = readOnly; - numMemObjectsInQueue_++; -} - -void VirtualGPU::MemoryDependency::clear(bool all) { - if (numMemObjectsInQueue_ > 0) { - if (all) { - endMemObjectsInQueue_ = numMemObjectsInQueue_; - } - - // If the current launch didn't start from the beginning, then move the data - if (0 != endMemObjectsInQueue_) { - size_t i, j; - // Preserve all objects from the current kernel - for (i = 0, j = endMemObjectsInQueue_; j < numMemObjectsInQueue_; i++, j++) { - memObjectsInQueue_[i].start_ = memObjectsInQueue_[j].start_; - memObjectsInQueue_[i].end_ = memObjectsInQueue_[j].end_; - memObjectsInQueue_[i].readOnly_ = memObjectsInQueue_[j].readOnly_; - } - } else if (numMemObjectsInQueue_ >= maxMemObjectsInQueue_) { - // note: The array growth shouldn't occur under the normal conditions, - // but in a case when SVM path sends the amount of SVM ptrs over - // the max size of kernel arguments - MemoryState* ptr = new MemoryState[maxMemObjectsInQueue_ << 1]; - if (nullptr == ptr) { - numMemObjectsInQueue_ = 0; - return; - } - maxMemObjectsInQueue_ <<= 1; - memcpy(ptr, memObjectsInQueue_, sizeof(MemoryState) * numMemObjectsInQueue_); - delete[] memObjectsInQueue_; - memObjectsInQueue_= ptr; - } - numMemObjectsInQueue_ -= endMemObjectsInQueue_; - endMemObjectsInQueue_ = 0; - } -} - -VirtualGPU::DmaFlushMgmt::DmaFlushMgmt(const Device& dev) : cbWorkload_(0), dispatchSplitSize_(0) { - aluCnt_ = dev.info().simdPerCU_ * dev.info().simdWidth_ * dev.info().maxComputeUnits_; - maxDispatchWorkload_ = static_cast(dev.info().maxEngineClockFrequency_) * - // find time in us - dev.settings().maxWorkloadTime_ * aluCnt_; - resetCbWorkload(dev); -} - -void VirtualGPU::DmaFlushMgmt::resetCbWorkload(const Device& dev) { - cbWorkload_ = 0; - maxCbWorkload_ = static_cast(dev.info().maxEngineClockFrequency_) * - // find time in us - dev.settings().minWorkloadTime_ * aluCnt_; -} - -void VirtualGPU::DmaFlushMgmt::findSplitSize(const Device& dev, uint64_t threads, - uint instructions) { - uint64_t workload = threads * instructions; - if (maxDispatchWorkload_ < workload) { - dispatchSplitSize_ = static_cast(maxDispatchWorkload_ / instructions); - uint fullLoad = dev.info().maxComputeUnits_ * dev.info().preferredWorkGroupSize_; - if ((dispatchSplitSize_ % fullLoad) != 0) { - dispatchSplitSize_ = (dispatchSplitSize_ / fullLoad + 1) * fullLoad; - } - } else { - dispatchSplitSize_ = - (threads > dev.settings().workloadSplitSize_) ? dev.settings().workloadSplitSize_ : 0; - } -} - -bool VirtualGPU::DmaFlushMgmt::isCbReady(VirtualGPU& gpu, uint64_t threads, uint instructions) { - bool cbReady = false; - uint64_t workload = amd::alignUp(threads, 4 * aluCnt_) * instructions; - // Add current workload to the overall workload in the current DMA - cbWorkload_ += workload; - // Did it exceed maximum? - if (cbWorkload_ > maxCbWorkload_) { - // Reset DMA workload - cbWorkload_ = 0; - // Increase workload of the next DMA buffer by 50% - maxCbWorkload_ = maxCbWorkload_ * 3 / 2; - if (maxCbWorkload_ > maxDispatchWorkload_) { - maxCbWorkload_ = maxDispatchWorkload_; - } - cbReady = true; - } - return cbReady; -} - -bool VirtualGPU::gslOpen(uint nEngines, gslEngineDescriptor* engines, uint32_t rtCUs) { - // GSL device initialization - dev().PerformFullInitialization(); - - // Wait the event - m_waitType = dev().settings().syncObject_ ? CAL_WAIT_LOW_CPU_UTILIZATION : CAL_WAIT_POLLING; - - if (!open(&dev(), nEngines, engines, rtCUs)) { - return false; - } - - return true; -} - -void VirtualGPU::gslDestroy() { close(dev().getNative()); } - -void VirtualGPU::addXferWrite(Memory& memory) { - if (xferWriteBuffers_.size() > 7) { - dev().xferWrite().release(*this, *xferWriteBuffers_.front()); - xferWriteBuffers_.pop_front(); - } - - // Delay destruction - xferWriteBuffers_.push_back(&memory); -} - -void VirtualGPU::releaseXferWrite() { - for (auto& memory : xferWriteBuffers_) { - dev().xferWrite().release(*this, *memory); - } - xferWriteBuffers_.clear(); -} - -void VirtualGPU::addPinnedMem(amd::Memory* mem) { - if (NULL == findPinnedMem(mem->getHostMem(), mem->getSize())) { - if (pinnedMems_.size() > 7) { - pinnedMems_.front()->release(); - pinnedMems_.pop_front(); - } - - // Start operation, since we should release mem object - flushDMA(getGpuEvent(dev().getGpuMemory(mem)->gslResource())->engineId_); - - // Delay destruction - pinnedMems_.push_back(mem); - } -} - -void VirtualGPU::releasePinnedMem() { - for (auto& amdMemory : pinnedMems_) { - amdMemory->release(); - } - pinnedMems_.clear(); -} - -amd::Memory* VirtualGPU::findPinnedMem(void* addr, size_t size) { - for (auto& amdMemory : pinnedMems_) { - if ((amdMemory->getHostMem() == addr) && (size <= amdMemory->getSize())) { - return amdMemory; - } - } - return NULL; -} - -bool VirtualGPU::createVirtualQueue(uint deviceQueueSize) { - uint MinDeviceQueueSize = 16 * 1024; - deviceQueueSize = std::max(deviceQueueSize, MinDeviceQueueSize); - - maskGroups_ = deviceQueueSize / (512 * Ki); - maskGroups_ = (maskGroups_ == 0) ? 1 : maskGroups_; - - // Align the queue size for the multiple dispatch scheduler. - // Each thread works with 32 entries * maskGroups - uint extra = deviceQueueSize % (sizeof(AmdAqlWrap) * DeviceQueueMaskSize * maskGroups_); - if (extra != 0) { - deviceQueueSize += (sizeof(AmdAqlWrap) * DeviceQueueMaskSize * maskGroups_) - extra; - } - - if (deviceQueueSize_ == deviceQueueSize) { - return true; - } else { - //! @todo Temporarily keep the buffer mapped for debug purpose - if (NULL != schedParams_) { - schedParams_->unmap(this); - } - delete vqHeader_; - delete virtualQueue_; - delete schedParams_; - vqHeader_ = NULL; - virtualQueue_ = NULL; - schedParams_ = NULL; - schedParamIdx_ = 0; - deviceQueueSize_ = 0; - } - uint numSlots = deviceQueueSize / sizeof(AmdAqlWrap); - uint allocSize = deviceQueueSize; - - // Add the virtual queue header - allocSize += sizeof(AmdVQueueHeader); - allocSize = amd::alignUp(allocSize, sizeof(AmdAqlWrap)); - - uint argOffs = allocSize; - - // Add the kernel arguments and wait events - uint singleArgSize = amd::alignUp( - dev().info().maxParameterSize_ + 64 + dev().settings().numWaitEvents_ * sizeof(uint64_t), - sizeof(AmdAqlWrap)); - allocSize += singleArgSize * numSlots; - - uint eventsOffs = allocSize; - // Add the device events - allocSize += dev().settings().numDeviceEvents_ * sizeof(AmdEvent); - - uint eventMaskOffs = allocSize; - // Add mask array for events - allocSize += amd::alignUp(dev().settings().numDeviceEvents_, DeviceQueueMaskSize) / 8; - - uint slotMaskOffs = allocSize; - // Add mask array for AmdAqlWrap slots - allocSize += amd::alignUp(numSlots, DeviceQueueMaskSize) / 8; - - virtualQueue_ = new Memory(dev(), allocSize); - Resource::MemoryType type = (GPU_PRINT_CHILD_KERNEL == 0) ? Resource::Local : Resource::Remote; - if ((virtualQueue_ == NULL) || !virtualQueue_->create(type)) { - return false; - } - address ptr = reinterpret_cast
(virtualQueue_->map(this, Resource::WriteOnly)); - if (NULL == ptr) { - return false; - } - // Clear memory - memset(ptr, 0, allocSize); - uint64_t vaBase = virtualQueue_->vmAddress(); - AmdVQueueHeader* header = reinterpret_cast(ptr); - - // Initialize the virtual queue header - header->aql_slot_num = numSlots; - header->event_slot_num = dev().settings().numDeviceEvents_; - header->event_slot_mask = vaBase + eventMaskOffs; - header->event_slots = vaBase + eventsOffs; - header->aql_slot_mask = vaBase + slotMaskOffs; - header->wait_size = dev().settings().numWaitEvents_; - header->arg_size = dev().info().maxParameterSize_ + 64; - header->mask_groups = maskGroups_; - vqHeader_ = new AmdVQueueHeader; - if (NULL == vqHeader_) { - return false; - } - *vqHeader_ = *header; - - // Go over all slots and perform initialization - AmdAqlWrap* slots = reinterpret_cast(&header[1]); - for (uint i = 0; i < numSlots; ++i) { - uint64_t argStart = vaBase + argOffs + i * singleArgSize; - slots[i].aql.kernarg_address = reinterpret_cast(argStart); - slots[i].wait_list = argStart + dev().info().maxParameterSize_ + 64; - } - // Upload data back to local memory - if (GPU_PRINT_CHILD_KERNEL == 0) { - virtualQueue_->unmap(this); - } - - schedParams_ = new Memory(dev(), 64 * Ki); - if ((schedParams_ == NULL) || !schedParams_->create(Resource::RemoteUSWC)) { - return false; - } - - ptr = reinterpret_cast
(schedParams_->map(this)); - - deviceQueueSize_ = deviceQueueSize; - - return true; -} - -VirtualGPU::VirtualGPU(Device& device) - : device::VirtualDevice(device), - CALGSLContext(), - engineID_(MainEngine), - activeKernelDesc_(NULL), - gpuDevice_(static_cast(device)), - printfDbg_(NULL), - printfDbgHSA_(NULL), - tsCache_(NULL), - vmMems_(NULL), - numVmMems_(0), - dmaFlushMgmt_(device), - hwRing_(0), - readjustTimeGPU_(0), - currTs_(NULL), - vqHeader_(NULL), - virtualQueue_(NULL), - schedParams_(NULL), - schedParamIdx_(0), - deviceQueueSize_(0), - maskGroups_(1), - hsaQueueMem_(NULL), - profileEnabled_(false) { - memset(&cal_, 0, sizeof(CalVirtualDesc)); - for (uint i = 0; i < AllEngines; ++i) { - cal_.events_[i].invalidate(); - } - memset(&cal_.samplersState_, 0xff, sizeof(cal_.samplersState_)); - - // Note: Virtual GPU device creation must be a thread safe operation - index_ = gpuDevice_.numOfVgpus_++; - gpuDevice_.vgpus_.resize(gpuDevice_.numOfVgpus()); - gpuDevice_.vgpus_[index()] = this; -} - -bool VirtualGPU::create(bool profiling, uint rtCUs, uint deviceQueueSize, - amd::CommandQueue::Priority priority) { - device::BlitManager::Setup blitSetup; - gslEngineDescriptor engines[2]; - uint engineMask = 0; - uint32_t num = 0; - - if (index() >= GPU_MAX_COMMAND_QUEUES) { - // Cap the maximum number of concurrent Virtual GPUs. - return false; - } - - // Virtual GPU will have profiling enabled - state_.profiling_ = profiling; - - { - if (dev().engines().numComputeRings()) { - uint idx; - - if ((amd::CommandQueue::RealTimeDisabled == rtCUs) && - (priority == amd::CommandQueue::Priority::Normal)) { - idx = index() % dev().engines().numComputeRings(); - engineMask = dev().engines().getMask((gslEngineID)( - dev().isComputeRingIDForced() ? dev().getforcedComputeEngineID() - : (dev().getFirstAvailableComputeEngineID() + idx))); - - } else { - if ((priority == amd::CommandQueue::Priority::Medium) && - (amd::CommandQueue::RealTimeDisabled == rtCUs)) { - engineMask = dev().engines().getMask((gslEngineID)(GSL_ENGINEID_COMPUTE_MEDIUM_PRIORITY)); - } else { - if (priority == amd::CommandQueue::Priority::Medium) { - engineMask = dev().engines().getMask((gslEngineID)(GSL_ENGINEID_COMPUTE_RT1)); - } else { - engineMask = dev().engines().getMask((gslEngineID)(GSL_ENGINEID_COMPUTE_RT)); - } - } - //!@todo This is not a generic solution and - // may have issues with > 8 queues - idx = index() % (dev().engines().numComputeRings() + dev().engines().numComputeRingsRT()); - } - // hwRing_ should be set 0 if forced to have single scratch buffer - hwRing_ = (dev().settings().useSingleScratch_) ? 0 : idx; - - if (dev().canDMA()) { - // If only 1 DMA engine is available then use that one - if (dev().engines().numDMAEngines() < 2) { - engineMask |= dev().engines().getMask(GSL_ENGINEID_DRMDMA0); - } else if (index() & 0x1) { - engineMask |= dev().engines().getMask(GSL_ENGINEID_DRMDMA0); - } else { - engineMask |= dev().engines().getMask(GSL_ENGINEID_DRMDMA1); - } - } - } else { - engineMask = dev().engines().getMask(GSL_ENGINEID_3DCOMPUTE0); - if (dev().canDMA()) { - engineMask |= dev().engines().getMask(GSL_ENGINEID_DRMDMA0); - } - } - } - num = dev().engines().getRequested(engineMask, engines); - - // Open GSL context - if ((num == 0) || !gslOpen(num, engines, rtCUs)) { - return false; - } - - // Diable double copy optimization, - // since UAV read from nonlocal is fast enough - blitSetup.disableCopyBufferToImageOpt_ = true; - if (!allocConstantBuffers()) { - return false; - } - - // Create Printf class - printfDbg_ = new PrintfDbg(gpuDevice_); - if ((NULL == printfDbg_) || !printfDbg_->create()) { - delete printfDbg_; - LogError("Could not allocate debug buffer for printf()!"); - return false; - } - - // Create HSAILPrintf class - printfDbgHSA_ = new PrintfDbgHSA(gpuDevice_); - if (NULL == printfDbgHSA_) { - delete printfDbgHSA_; - LogError("Could not create PrintfDbgHSA class!"); - return false; - } - - // Choose the appropriate class for blit engine - switch (dev().settings().blitEngine_) { - default: - // Fall through ... - case Settings::BlitEngineHost: - blitSetup.disableAll(); - // Fall through ... - case Settings::BlitEngineCAL: - case Settings::BlitEngineKernel: - // use host blit for HW debug - if (dev().settings().enableHwDebug_) { - blitSetup.disableCopyImageToBuffer_ = true; - blitSetup.disableCopyBufferToImage_ = true; - } - blitMgr_ = new KernelBlitManager(*this, blitSetup); - break; - } - if ((NULL == blitMgr_) || !blitMgr_->create(gpuDevice_)) { - LogError("Could not create BlitManager!"); - return false; - } - - tsCache_ = new TimeStampCache(*this); - if (NULL == tsCache_) { - LogError("Could not create TimeStamp cache!"); - return false; - } - - if (!memoryDependency().create(dev().settings().numMemDependencies_)) { - LogError("Could not create the array of memory objects!"); - return false; - } - - if (!allocHsaQueueMem()) { - LogError("Could not create hsaQueueMem object!"); - return false; - } - - // Check if the app requested a device queue creation - if (dev().settings().useDeviceQueue_ && (0 != deviceQueueSize) && - !createVirtualQueue(deviceQueueSize)) { - LogError("Could not create a virtual queue!"); - return false; - } - - return true; -} - -bool VirtualGPU::allocHsaQueueMem() { - // Allocate a dummy HSA queue - hsaQueueMem_ = new Memory(dev(), sizeof(amd_queue_t)); - if ((hsaQueueMem_ == NULL) || (!hsaQueueMem_->create(Resource::Local))) { - delete hsaQueueMem_; - return false; - } - amd_queue_t* queue = reinterpret_cast(hsaQueueMem_->map(NULL, Resource::WriteOnly)); - if (NULL == queue) { - delete hsaQueueMem_; - return false; - } - memset(queue, 0, sizeof(amd_queue_t)); - // Provide private and local heap addresses - const static uint addressShift = LP64_SWITCH(0, 32); - queue->private_segment_aperture_base_hi = - static_cast(dev().gslCtx()->getPrivateApertureBase() >> addressShift); - queue->group_segment_aperture_base_hi = - static_cast(dev().gslCtx()->getSharedApertureBase() >> addressShift); - hsaQueueMem_->unmap(NULL); - return true; -} - -VirtualGPU::~VirtualGPU() { - // Not safe to remove a queue. So lock the device - amd::ScopedLock k(dev().lockAsyncOps()); - amd::ScopedLock lock(dev().vgpusAccess()); - - uint i; - // Destroy all kernels - for (const auto& it : gslKernels_) { - if (it.first != 0) { - freeKernelDesc(it.second); - } - } - gslKernels_.clear(); - - // Destroy all memories - static const bool SkipScratch = false; - releaseMemObjects(SkipScratch); - - // Destroy printf object - delete printfDbg_; - - // Destroy printfHSA object - delete printfDbgHSA_; - - // Destroy BlitManager object - delete blitMgr_; - - // Destroy TimeStamp cache - delete tsCache_; - - // Destroy resource list with the constant buffers - for (i = 0; i < constBufs_.size(); ++i) { - delete constBufs_[i]; - } - - gslDestroy(); - - gpuDevice_.numOfVgpus_--; - gpuDevice_.vgpus_.erase(gpuDevice_.vgpus_.begin() + index()); - for (uint idx = index(); idx < dev().vgpus().size(); ++idx) { - dev().vgpus()[idx]->index_--; - } - - // Release scratch buffer memory to reduce memory pressure - //!@note OCLtst uses single device with multiple tests - //! Release memory only if it's the last command queue. - //! The first queue is reserved for the transfers on device - if (gpuDevice_.numOfVgpus_ <= 1) { - gpuDevice_.destroyScratchBuffers(); - } - - delete[] vmMems_; - //! @todo Temporarily keep the buffer mapped for debug purpose - if (NULL != schedParams_) { - schedParams_->unmap(this); - } - delete vqHeader_; - delete virtualQueue_; - delete schedParams_; - delete hsaQueueMem_; -} - -void VirtualGPU::submitReadMemory(amd::ReadMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - // Translate memory references and ensure cache up-to-date - gpu::Memory* memory = dev().getGpuMemory(&vcmd.source()); - - size_t offset = 0; - // Find if virtual address is a CL allocation - device::Memory* hostMemory = dev().findMemoryFromVA(vcmd.destination(), &offset); - - profilingBegin(vcmd, true); - - memory->syncCacheFromHost(*this); - cl_command_type type = vcmd.type(); - bool result = false; - amd::Memory* bufferFromImage = NULL; - - // Force buffer read for IMAGE1D_BUFFER - if ((type == CL_COMMAND_READ_IMAGE) && - (vcmd.source().getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - bufferFromImage = createBufferFromImage(vcmd.source()); - if (NULL == bufferFromImage) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - type = CL_COMMAND_READ_BUFFER; - memory = dev().getGpuMemory(bufferFromImage); - } - } - - // Process different write commands - switch (type) { - case CL_COMMAND_READ_BUFFER: { - amd::Coord3D origin(vcmd.origin()[0]); - amd::Coord3D size(vcmd.size()[0]); - if (NULL != bufferFromImage) { - size_t elemSize = vcmd.source().asImage()->getImageFormat().getElementSize(); - origin.c[0] *= elemSize; - size.c[0] *= elemSize; - } - if (hostMemory != NULL) { - // Accelerated transfer without pinning - amd::Coord3D dstOrigin(offset); - result = blitMgr().copyBuffer(*memory, *hostMemory, origin, dstOrigin, size, - vcmd.isEntireMemory()); - } else { - // The logic below will perform 2 step copy to make sure memory pinning doesn't - // occur on the first unaligned page, because in Windows memory manager can - // have CPU access to the allocation header in another thread - // and a race condition is possible. - char* tmpHost = - amd::alignUp(reinterpret_cast(vcmd.destination()), PinnedMemoryAlignment); - - // Find the partial size for unaligned copy - size_t partial = tmpHost - reinterpret_cast(vcmd.destination()); - result = true; - // Check if it's staging copy, then ignore unaligned address - if (size[0] <= dev().settings().pinnedMinXferSize_) { - partial = size[0]; - } - // Make first step transfer - if (partial > 0) { - result = blitMgr().readBuffer(*memory, vcmd.destination(), origin, partial); - } - // Second step transfer if something left to copy - if (partial < size[0]) { - result &= blitMgr().readBuffer(*memory, tmpHost, origin[0] + partial, size[0] - partial); - } - } - if (NULL != bufferFromImage) { - bufferFromImage->release(); - } - } break; - case CL_COMMAND_READ_BUFFER_RECT: { - amd::BufferRect hostbufferRect; - amd::Coord3D region(0); - amd::Coord3D hostOrigin(vcmd.hostRect().start_ + offset); - hostbufferRect.create(hostOrigin.c, vcmd.size().c, vcmd.hostRect().rowPitch_, - vcmd.hostRect().slicePitch_); - if (hostMemory != NULL) { - result = blitMgr().copyBufferRect(*memory, *hostMemory, vcmd.bufRect(), hostbufferRect, - vcmd.size(), vcmd.isEntireMemory()); - } else { - result = blitMgr().readBufferRect(*memory, vcmd.destination(), vcmd.bufRect(), - vcmd.hostRect(), vcmd.size(), vcmd.isEntireMemory()); - } - } break; - case CL_COMMAND_READ_IMAGE: - if (hostMemory != NULL) { - // Accelerated image to buffer transfer without pinning - amd::Coord3D dstOrigin(offset); - result = - blitMgr().copyImageToBuffer(*memory, *hostMemory, vcmd.origin(), dstOrigin, vcmd.size(), - vcmd.isEntireMemory(), vcmd.rowPitch(), vcmd.slicePitch()); - } else { - result = blitMgr().readImage(*memory, vcmd.destination(), vcmd.origin(), vcmd.size(), - vcmd.rowPitch(), vcmd.slicePitch(), vcmd.isEntireMemory()); - } - break; - default: - LogError("Unsupported type for the read command"); - break; - } - - if (!result) { - LogError("submitReadMemory failed!"); - vcmd.setStatus(CL_INVALID_OPERATION); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitWriteMemory(amd::WriteMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - // Translate memory references and ensure cache up to date - gpu::Memory* memory = dev().getGpuMemory(&vcmd.destination()); - size_t offset = 0; - // Find if virtual address is a CL allocation - device::Memory* hostMemory = dev().findMemoryFromVA(vcmd.source(), &offset); - - profilingBegin(vcmd, true); - - bool entire = vcmd.isEntireMemory(); - - // Synchronize memory from host if necessary - device::Memory::SyncFlags syncFlags; - syncFlags.skipEntire_ = entire; - memory->syncCacheFromHost(*this, syncFlags); - - cl_command_type type = vcmd.type(); - bool result = false; - amd::Memory* bufferFromImage = NULL; - - // Force buffer write for IMAGE1D_BUFFER - if ((type == CL_COMMAND_WRITE_IMAGE) && - (vcmd.destination().getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - bufferFromImage = createBufferFromImage(vcmd.destination()); - if (NULL == bufferFromImage) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - type = CL_COMMAND_WRITE_BUFFER; - memory = dev().getGpuMemory(bufferFromImage); - } - } - - // Process different write commands - switch (type) { - case CL_COMMAND_WRITE_BUFFER: { - amd::Coord3D origin(vcmd.origin()[0]); - amd::Coord3D size(vcmd.size()[0]); - if (NULL != bufferFromImage) { - size_t elemSize = vcmd.destination().asImage()->getImageFormat().getElementSize(); - origin.c[0] *= elemSize; - size.c[0] *= elemSize; - } - if (hostMemory != NULL) { - // Accelerated transfer without pinning - amd::Coord3D srcOrigin(offset); - result = blitMgr().copyBuffer(*hostMemory, *memory, srcOrigin, origin, size, - vcmd.isEntireMemory()); - } else { - // The logic below will perform 2 step copy to make sure memory pinning doesn't - // occur on the first unaligned page, because in Windows memory manager can - // have CPU access to the allocation header in another thread - // and a race condition is possible. - const char* tmpHost = - amd::alignUp(reinterpret_cast(vcmd.source()), PinnedMemoryAlignment); - - // Find the partial size for unaligned copy - size_t partial = tmpHost - reinterpret_cast(vcmd.source()); - result = true; - // Check if it's staging copy, then ignore unaligned address - if (size[0] <= dev().settings().pinnedMinXferSize_) { - partial = size[0]; - } - // Make first step transfer - if (partial > 0) { - result = blitMgr().writeBuffer(vcmd.source(), *memory, origin, partial); - } - // Second step transfer if something left to copy - if (partial < size[0]) { - result &= blitMgr().writeBuffer(tmpHost, *memory, origin[0] + partial, size[0] - partial); - } - } - if (NULL != bufferFromImage) { - bufferFromImage->release(); - } - } break; - case CL_COMMAND_WRITE_BUFFER_RECT: { - amd::BufferRect hostbufferRect; - amd::Coord3D region(0); - amd::Coord3D hostOrigin(vcmd.hostRect().start_ + offset); - hostbufferRect.create(hostOrigin.c, vcmd.size().c, vcmd.hostRect().rowPitch_, - vcmd.hostRect().slicePitch_); - if (hostMemory != NULL) { - result = blitMgr().copyBufferRect(*hostMemory, *memory, hostbufferRect, vcmd.bufRect(), - vcmd.size(), vcmd.isEntireMemory()); - } else { - result = blitMgr().writeBufferRect(vcmd.source(), *memory, vcmd.hostRect(), vcmd.bufRect(), - vcmd.size(), vcmd.isEntireMemory()); - } - } break; - case CL_COMMAND_WRITE_IMAGE: - if (hostMemory != NULL) { - // Accelerated buffer to image transfer without pinning - amd::Coord3D srcOrigin(offset); - result = - blitMgr().copyBufferToImage(*hostMemory, *memory, srcOrigin, vcmd.origin(), vcmd.size(), - vcmd.isEntireMemory(), vcmd.rowPitch(), vcmd.slicePitch()); - } else { - result = blitMgr().writeImage(vcmd.source(), *memory, vcmd.origin(), vcmd.size(), - vcmd.rowPitch(), vcmd.slicePitch(), vcmd.isEntireMemory()); - } - break; - default: - LogError("Unsupported type for the write command"); - break; - } - - if (!result) { - LogError("submitWriteMemory failed!"); - vcmd.setStatus(CL_INVALID_OPERATION); - } else { - // Mark this as the most-recently written cache of the destination - vcmd.destination().signalWrite(&gpuDevice_); - } - profilingEnd(vcmd); -} - -bool VirtualGPU::copyMemory(cl_command_type type, amd::Memory& srcMem, amd::Memory& dstMem, - bool entire, const amd::Coord3D& srcOrigin, - const amd::Coord3D& dstOrigin, const amd::Coord3D& size, - const amd::BufferRect& srcRect, const amd::BufferRect& dstRect) { - // Translate memory references and ensure cache up-to-date - gpu::Memory* dstMemory = dev().getGpuMemory(&dstMem); - gpu::Memory* srcMemory = dev().getGpuMemory(&srcMem); - - // Synchronize source and destination memory - device::Memory::SyncFlags syncFlags; - syncFlags.skipEntire_ = entire; - dstMemory->syncCacheFromHost(*this, syncFlags); - srcMemory->syncCacheFromHost(*this); - - amd::Memory* bufferFromImageSrc = NULL; - amd::Memory* bufferFromImageDst = NULL; - - // Force buffer read for IMAGE1D_BUFFER - if ((srcMem.getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - bufferFromImageSrc = createBufferFromImage(srcMem); - if (NULL == bufferFromImageSrc) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - type = CL_COMMAND_COPY_BUFFER; - srcMemory = dev().getGpuMemory(bufferFromImageSrc); - } - } - // Force buffer write for IMAGE1D_BUFFER - if ((dstMem.getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - bufferFromImageDst = createBufferFromImage(dstMem); - if (NULL == bufferFromImageDst) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - type = CL_COMMAND_COPY_BUFFER; - dstMemory = dev().getGpuMemory(bufferFromImageDst); - } - } - - bool result = false; - - // Check if HW can be used for memory copy - switch (type) { - case CL_COMMAND_SVM_MEMCPY: - case CL_COMMAND_COPY_BUFFER: { - amd::Coord3D realSrcOrigin(srcOrigin[0]); - amd::Coord3D realDstOrigin(dstOrigin[0]); - amd::Coord3D realSize(size.c[0], size.c[1], size.c[2]); - - if (NULL != bufferFromImageSrc) { - size_t elemSize = srcMem.asImage()->getImageFormat().getElementSize(); - realSrcOrigin.c[0] *= elemSize; - if (NULL != bufferFromImageDst) { - realDstOrigin.c[0] *= elemSize; - } - realSize.c[0] *= elemSize; - } else if (NULL != bufferFromImageDst) { - size_t elemSize = dstMem.asImage()->getImageFormat().getElementSize(); - realDstOrigin.c[0] *= elemSize; - realSize.c[0] *= elemSize; - } - - result = blitMgr().copyBuffer(*srcMemory, *dstMemory, realSrcOrigin, realDstOrigin, realSize, - entire); - - if (NULL != bufferFromImageSrc) { - bufferFromImageSrc->release(); - } - if (NULL != bufferFromImageDst) { - bufferFromImageDst->release(); - } - } break; - case CL_COMMAND_COPY_BUFFER_RECT: - result = blitMgr().copyBufferRect(*srcMemory, *dstMemory, srcRect, dstRect, size, entire); - break; - case CL_COMMAND_COPY_IMAGE_TO_BUFFER: - result = - blitMgr().copyImageToBuffer(*srcMemory, *dstMemory, srcOrigin, dstOrigin, size, entire); - break; - case CL_COMMAND_COPY_BUFFER_TO_IMAGE: - result = - blitMgr().copyBufferToImage(*srcMemory, *dstMemory, srcOrigin, dstOrigin, size, entire); - break; - case CL_COMMAND_COPY_IMAGE: - result = blitMgr().copyImage(*srcMemory, *dstMemory, srcOrigin, dstOrigin, size, entire); - break; - default: - LogError("Unsupported command type for memory copy!"); - break; - } - - if (!result) { - LogError("submitCopyMemory failed!"); - return false; - } else { - // Mark this as the most-recently written cache of the destination - dstMem.signalWrite(&gpuDevice_); - } - return true; -} - -void VirtualGPU::submitCopyMemory(amd::CopyMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd); - - cl_command_type type = vcmd.type(); - bool entire = vcmd.isEntireMemory(); - - if (!copyMemory(type, vcmd.source(), vcmd.destination(), entire, vcmd.srcOrigin(), - vcmd.dstOrigin(), vcmd.size(), vcmd.srcRect(), vcmd.dstRect())) { - vcmd.setStatus(CL_INVALID_OPERATION); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - profilingBegin(vcmd); - - cl_command_type type = vcmd.type(); - // no op for FGS supported device - if (!dev().isFineGrainedSystem()) { - amd::Coord3D srcOrigin(0, 0, 0); - amd::Coord3D dstOrigin(0, 0, 0); - amd::Coord3D size(vcmd.srcSize(), 1, 1); - amd::BufferRect srcRect; - amd::BufferRect dstRect; - - bool result = false; - amd::Memory* srcMem = amd::MemObjMap::FindMemObj(vcmd.src()); - amd::Memory* dstMem = amd::MemObjMap::FindMemObj(vcmd.dst()); - - device::Memory::SyncFlags syncFlags; - if (nullptr != srcMem) { - srcMem->commitSvmMemory(); - srcOrigin.c[0] = - static_cast(vcmd.src()) - static_cast
(srcMem->getSvmPtr()); - if (!(srcMem->validateRegion(srcOrigin, size))) { - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - } - if (nullptr != dstMem) { - dstMem->commitSvmMemory(); - dstOrigin.c[0] = - static_cast(vcmd.dst()) - static_cast
(dstMem->getSvmPtr()); - if (!(dstMem->validateRegion(dstOrigin, size))) { - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - } - - if (nullptr == srcMem && nullptr == dstMem) { // both not in svm space - amd::Os::fastMemcpy(vcmd.dst(), vcmd.src(), vcmd.srcSize()); - result = true; - } else if (nullptr == srcMem && nullptr != dstMem) { // src not in svm space - Memory* memory = dev().getGpuMemory(dstMem); - // Synchronize source and destination memory - syncFlags.skipEntire_ = dstMem->isEntirelyCovered(dstOrigin, size); - memory->syncCacheFromHost(*this, syncFlags); - - result = blitMgr().writeBuffer(vcmd.src(), *memory, dstOrigin, size, - dstMem->isEntirelyCovered(dstOrigin, size)); - // Mark this as the most-recently written cache of the destination - dstMem->signalWrite(&gpuDevice_); - } else if (nullptr != srcMem && nullptr == dstMem) { // dst not in svm space - Memory* memory = dev().getGpuMemory(srcMem); - // Synchronize source and destination memory - memory->syncCacheFromHost(*this); - - result = blitMgr().readBuffer(*memory, vcmd.dst(), srcOrigin, size, - srcMem->isEntirelyCovered(srcOrigin, size)); - } else if (nullptr != srcMem && nullptr != dstMem) { // both in svm space - bool entire = - srcMem->isEntirelyCovered(srcOrigin, size) && dstMem->isEntirelyCovered(dstOrigin, size); - result = - copyMemory(type, *srcMem, *dstMem, entire, srcOrigin, dstOrigin, size, srcRect, dstRect); - } - - if (!result) { - vcmd.setStatus(CL_INVALID_OPERATION); - } - } else { - // direct memcpy for FGS enabled system - amd::SvmBuffer::memFill(vcmd.dst(), vcmd.src(), vcmd.srcSize(), 1); - } - profilingEnd(vcmd); -} - -void VirtualGPU::submitMapMemory(amd::MapMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd, true); - - gpu::Memory* memory = dev().getGpuMemory(&vcmd.memory()); - - // Save map info for unmap operation - memory->saveMapInfo(vcmd.mapPtr(), vcmd.origin(), vcmd.size(), vcmd.mapFlags(), - vcmd.isEntireMemory()); - - // If we have host memory, use it - if ((memory->owner()->getHostMem() != NULL) && memory->isDirectMap()) { - if (!memory->isHostMemDirectAccess()) { - // Make sure GPU finished operation before - // synchronization with the backing store - memory->wait(*this); - } - - // Target is the backing store, so just ensure that owner is up-to-date - memory->owner()->cacheWriteBack(); - - // Add memory to VA cache, so rutnime can detect direct access to VA - dev().addVACache(memory); - } else if (memory->isPersistentDirectMap()) { - // Nothing to do here - } else if (memory->mapMemory() != NULL) { - // Target is a remote resource, so copy - assert(memory->mapMemory() != NULL); - if (vcmd.mapFlags() & (CL_MAP_READ | CL_MAP_WRITE)) { - amd::Coord3D dstOrigin(0, 0, 0); - if (memory->cal()->buffer_) { - if (!blitMgr().copyBuffer(*memory, *memory->mapMemory(), vcmd.origin(), vcmd.origin(), - vcmd.size(), vcmd.isEntireMemory())) { - LogError("submitMapMemory() - copy failed"); - vcmd.setStatus(CL_MAP_FAILURE); - } - } else if ((vcmd.memory().getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - Memory* memoryBuf = memory; - amd::Coord3D origin(vcmd.origin()[0]); - amd::Coord3D size(vcmd.size()[0]); - size_t elemSize = vcmd.memory().asImage()->getImageFormat().getElementSize(); - origin.c[0] *= elemSize; - size.c[0] *= elemSize; - - amd::Memory* bufferFromImage = createBufferFromImage(vcmd.memory()); - if (NULL == bufferFromImage) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - memoryBuf = dev().getGpuMemory(bufferFromImage); - } - if (!blitMgr().copyBuffer(*memoryBuf, *memory->mapMemory(), origin, dstOrigin, size, - vcmd.isEntireMemory())) { - LogError("submitMapMemory() - copy failed"); - vcmd.setStatus(CL_MAP_FAILURE); - } - if (NULL != bufferFromImage) { - bufferFromImage->release(); - } - } else { - // Validate if it's a view for a map of mip level - if (vcmd.memory().parent() != NULL) { - amd::Image* amdImage = vcmd.memory().parent()->asImage(); - if ((amdImage != NULL) && (amdImage->getMipLevels() > 1)) { - // Save map write info in the parent object - dev().getGpuMemory(amdImage)->saveMapInfo(vcmd.mapPtr(), vcmd.origin(), vcmd.size(), - vcmd.mapFlags(), vcmd.isEntireMemory(), - vcmd.memory().asImage()); - } - } - if (!blitMgr().copyImageToBuffer(*memory, *memory->mapMemory(), vcmd.origin(), dstOrigin, - vcmd.size(), vcmd.isEntireMemory())) { - LogError("submitMapMemory() - copy failed"); - vcmd.setStatus(CL_MAP_FAILURE); - } - } - } - } else { - LogError("Unhandled map!"); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitUnmapMemory(amd::UnmapMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - gpu::Memory* memory = dev().getGpuMemory(&vcmd.memory()); - amd::Memory* owner = memory->owner(); - bool unmapMip = false; - const device::Memory::WriteMapInfo* writeMapInfo = memory->writeMapInfo(vcmd.mapPtr()); - if (nullptr == writeMapInfo) { - LogError("Unmap without map call"); - return; - } - profilingBegin(vcmd, true); - - // Check if image is a mipmap and assign a saved view - amd::Image* amdImage = owner->asImage(); - if ((amdImage != NULL) && (amdImage->getMipLevels() > 1) && (writeMapInfo->baseMip_ != NULL)) { - // Assign mip level view - amdImage = writeMapInfo->baseMip_; - // Clear unmap flags from the parent image - memory->clearUnmapInfo(vcmd.mapPtr()); - memory = dev().getGpuMemory(amdImage); - unmapMip = true; - writeMapInfo = memory->writeMapInfo(vcmd.mapPtr()); - } - - // We used host memory - if ((owner->getHostMem() != NULL) && memory->isDirectMap()) { - if (writeMapInfo->isUnmapWrite()) { - // Target is the backing store, so sync - owner->signalWrite(NULL); - memory->syncCacheFromHost(*this); - } - // Remove memory from VA cache - dev().removeVACache(memory); - } - // data check was added for persistent memory that failed to get aperture - // and therefore are treated like a remote resource - else if (memory->isPersistentDirectMap() && (memory->data() != NULL)) { - memory->unmap(this); - } else if (memory->mapMemory() != NULL) { - if (writeMapInfo->isUnmapWrite()) { - amd::Coord3D srcOrigin(0, 0, 0); - // Target is a remote resource, so copy - assert(memory->mapMemory() != NULL); - if (memory->cal()->buffer_) { - if (!blitMgr().copyBuffer(*memory->mapMemory(), *memory, writeMapInfo->origin_, - writeMapInfo->origin_, writeMapInfo->region_, - writeMapInfo->isEntire())) { - LogError("submitUnmapMemory() - copy failed"); - vcmd.setStatus(CL_OUT_OF_RESOURCES); - } - } else if ((vcmd.memory().getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - Memory* memoryBuf = memory; - amd::Coord3D origin(writeMapInfo->origin_[0]); - amd::Coord3D size(writeMapInfo->region_[0]); - size_t elemSize = vcmd.memory().asImage()->getImageFormat().getElementSize(); - origin.c[0] *= elemSize; - size.c[0] *= elemSize; - - amd::Memory* bufferFromImage = createBufferFromImage(vcmd.memory()); - if (NULL == bufferFromImage) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - memoryBuf = dev().getGpuMemory(bufferFromImage); - } - if (!blitMgr().copyBuffer(*memory->mapMemory(), *memoryBuf, srcOrigin, origin, size, - writeMapInfo->isEntire())) { - LogError("submitUnmapMemory() - copy failed"); - vcmd.setStatus(CL_OUT_OF_RESOURCES); - } - if (NULL != bufferFromImage) { - bufferFromImage->release(); - } - } else { - if (!blitMgr().copyBufferToImage(*memory->mapMemory(), *memory, srcOrigin, - writeMapInfo->origin_, writeMapInfo->region_, - writeMapInfo->isEntire())) { - LogError("submitUnmapMemory() - copy failed"); - vcmd.setStatus(CL_OUT_OF_RESOURCES); - } - } - } - } else { - LogError("Unhandled unmap!"); - vcmd.setStatus(CL_INVALID_VALUE); - } - - // Clear unmap flags - memory->clearUnmapInfo(vcmd.mapPtr()); - - // Release a view for a mipmap map - if (unmapMip) { - amdImage->release(); - } - profilingEnd(vcmd); -} - -bool VirtualGPU::fillMemory(cl_command_type type, amd::Memory* amdMemory, const void* pattern, - size_t patternSize, const amd::Coord3D& origin, - const amd::Coord3D& size, bool forceBlit) { - gpu::Memory* memory = dev().getGpuMemory(amdMemory); - bool entire = amdMemory->isEntirelyCovered(origin, size); - - // Synchronize memory from host if necessary - device::Memory::SyncFlags syncFlags; - syncFlags.skipEntire_ = entire; - memory->syncCacheFromHost(*this, syncFlags); - - bool result = false; - amd::Memory* bufferFromImage = NULL; - float fillValue[4]; - - // Force fill buffer for IMAGE1D_BUFFER - if ((type == CL_COMMAND_FILL_IMAGE) && (amdMemory->getType() == CL_MEM_OBJECT_IMAGE1D_BUFFER)) { - bufferFromImage = createBufferFromImage(*amdMemory); - if (NULL == bufferFromImage) { - LogError("We should not fail buffer creation from image_buffer!"); - } else { - type = CL_COMMAND_FILL_BUFFER; - memory = dev().getGpuMemory(bufferFromImage); - } - } - - // Find the the right fill operation - switch (type) { - case CL_COMMAND_FILL_BUFFER: - case CL_COMMAND_SVM_MEMFILL: { - amd::Coord3D realOrigin(origin[0]); - amd::Coord3D realSize(size[0]); - // Reprogram fill parameters if it's an IMAGE1D_BUFFER object - if (NULL != bufferFromImage) { - size_t elemSize = amdMemory->asImage()->getImageFormat().getElementSize(); - realOrigin.c[0] *= elemSize; - realSize.c[0] *= elemSize; - memset(fillValue, 0, sizeof(fillValue)); - amdMemory->asImage()->getImageFormat().formatColor(pattern, fillValue); - pattern = fillValue; - patternSize = elemSize; - } - result = blitMgr().fillBuffer(*memory, pattern, patternSize, realOrigin, realSize, - amdMemory->isEntirelyCovered(origin, size), forceBlit); - if (NULL != bufferFromImage) { - bufferFromImage->release(); - } - } break; - case CL_COMMAND_FILL_IMAGE: - result = blitMgr().fillImage(*memory, pattern, origin, size, - amdMemory->isEntirelyCovered(origin, size)); - break; - default: - LogError("Unsupported command type for FillMemory!"); - break; - } - - if (!result) { - LogError("fillMemory failed!"); - return false; - } - - // Mark this as the most-recently written cache of the destination - amdMemory->signalWrite(&gpuDevice_); - return true; -} - -void VirtualGPU::submitFillMemory(amd::FillMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd, true); - - if (!fillMemory(vcmd.type(), &vcmd.memory(), vcmd.pattern(), vcmd.patternSize(), vcmd.origin(), - vcmd.size())) { - vcmd.setStatus(CL_INVALID_OPERATION); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitSvmMapMemory(amd::SvmMapMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd, true); - - // no op for FGS supported device - if (!dev().isFineGrainedSystem()) { - // Make sure we have memory for the command execution - gpu::Memory* memory = dev().getGpuMemory(vcmd.getSvmMem()); - memory->saveMapInfo(vcmd.svmPtr(), vcmd.origin(), vcmd.size(), vcmd.mapFlags(), - vcmd.isEntireMemory()); - - if (memory->mapMemory() != NULL) { - if (vcmd.mapFlags() & (CL_MAP_READ | CL_MAP_WRITE)) { - assert(memory->cal()->buffer_ && "SVM memory can't be an image"); - if (!blitMgr().copyBuffer(*memory, *memory->mapMemory(), vcmd.origin(), vcmd.origin(), - vcmd.size(), vcmd.isEntireMemory())) { - LogError("submitSVMMapMemory() - copy failed"); - vcmd.setStatus(CL_MAP_FAILURE); - } - } - } else if ((memory->owner()->getHostMem() != nullptr) && memory->isDirectMap()) { - if (!memory->isHostMemDirectAccess()) { - // Make sure GPU finished operation before - // synchronization with the backing store - memory->wait(*this); - } - - // Target is the backing store, so just ensure that owner is up-to-date - memory->owner()->cacheWriteBack(); - } else { - LogError("Unhandled svm map!"); - } - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - profilingBegin(vcmd, true); - - // no op for FGS supported device - if (!dev().isFineGrainedSystem()) { - gpu::Memory* memory = dev().getGpuMemory(vcmd.getSvmMem()); - const device::Memory::WriteMapInfo* writeMapInfo = memory->writeMapInfo(vcmd.svmPtr()); - - if (memory->mapMemory() != NULL) { - if (writeMapInfo->isUnmapWrite()) { - // Target is a remote resource, so copy - assert(memory->cal()->buffer_ && "SVM memory can't be an image"); - if (!blitMgr().copyBuffer(*memory->mapMemory(), *memory, writeMapInfo->origin_, - writeMapInfo->origin_, writeMapInfo->region_, - writeMapInfo->isEntire())) { - LogError("submitSvmUnmapMemory() - copy failed"); - vcmd.setStatus(CL_OUT_OF_RESOURCES); - } - } - } else if ((memory->owner()->getHostMem() != nullptr) && memory->isDirectMap()) { - if (writeMapInfo->isUnmapWrite()) { - // Target is the backing store, so sync - memory->owner()->signalWrite(nullptr); - memory->syncCacheFromHost(*this); - } - } - memory->clearUnmapInfo(vcmd.svmPtr()); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd, true); - - if (!dev().isFineGrainedSystem()) { - size_t patternSize = vcmd.patternSize(); - size_t fillSize = patternSize * vcmd.times(); - size_t offset = 0; - amd::Memory* dstMemory = amd::MemObjMap::FindMemObj(vcmd.dst()); - assert(dstMemory && "No svm Buffer to fill with!"); - offset = reinterpret_cast(vcmd.dst()) - - reinterpret_cast(dstMemory->getSvmPtr()); - assert((offset >= 0) && "wrong svm ptr to fill with!"); - - gpu::Memory* memory = dev().getGpuMemory(dstMemory); - - amd::Coord3D origin(offset, 0, 0); - amd::Coord3D size(fillSize, 1, 1); - assert((dstMemory->validateRegion(origin, size)) && "The incorrect fill size!"); - - if (!fillMemory(vcmd.type(), dstMemory, vcmd.pattern(), vcmd.patternSize(), origin, size)) { - vcmd.setStatus(CL_INVALID_OPERATION); - } - // Mark this as the most-recently written cache of the destination - dstMemory->signalWrite(&gpuDevice_); - } else { - // for FGS capable device, fill CPU memory directly - amd::SvmBuffer::memFill(vcmd.dst(), vcmd.pattern(), vcmd.patternSize(), vcmd.times()); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitMigrateMemObjects(amd::MigrateMemObjectsCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd, true); - - for (const auto& it : vcmd.memObjects()) { - // Find device memory - gpu::Memory* memory = dev().getGpuMemory(it); - - if (vcmd.migrationFlags() & CL_MIGRATE_MEM_OBJECT_HOST) { - memory->mgpuCacheWriteBack(); - } else if (vcmd.migrationFlags() & CL_MIGRATE_MEM_OBJECT_CONTENT_UNDEFINED) { - // Synchronize memory from host if necessary. - // The sync function will perform memory migration from - // another device if necessary - device::Memory::SyncFlags syncFlags; - memory->syncCacheFromHost(*this, syncFlags); - } else { - LogWarning("Unknown operation for memory migration!"); - } - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitSvmFreeMemory(amd::SvmFreeMemoryCommand& vcmd) { - // in-order semantics: previous commands need to be done before we start - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd); - std::vector& svmPointers = vcmd.svmPointers(); - if (vcmd.pfnFreeFunc() == NULL) { - // pointers allocated using clSVMAlloc - for (uint32_t i = 0; i < svmPointers.size(); ++i) { - dev().svmFree(svmPointers[i]); - } - } else { - vcmd.pfnFreeFunc()(as_cl(vcmd.queue()->asCommandQueue()), svmPointers.size(), - static_cast(&(svmPointers[0])), vcmd.userData()); - } - profilingEnd(vcmd); -} - -void VirtualGPU::findIterations(const amd::NDRangeContainer& sizes, const amd::NDRange& local, - amd::NDRange& groups, amd::NDRange& remainder, size_t& extra) { - size_t dimensions = sizes.dimensions(); - - if (cal()->iterations_ > 1) { - size_t iterations = cal()->iterations_; - cal_.iterations_ = 1; - - // Find the total amount of all groups - groups = sizes.global() / local; - for (uint j = 0; j < dimensions; ++j) { - if ((sizes.global()[j] % local[j]) != 0) { - groups[j]++; - } - } - - // Calculate the real number of required iterations and - // the workgroup size of each iteration - for (int j = (dimensions - 1); j >= 0; --j) { - // Find possible size of each iteration - size_t tmp = (groups[j] / iterations); - // Make sure the group size is more than 1 - if (tmp > 0) { - remainder = groups; - remainder[j] = (groups[j] % tmp); - - extra = ((groups[j] / tmp) + - // Check for the remainder - ((remainder[j] != 0) ? 1 : 0)); - // Recalculate the number of iterations - cal_.iterations_ *= extra; - if (remainder[j] == 0) { - extra = 0; - } - groups[j] = tmp; - break; - } else { - iterations = ((iterations / groups[j]) + (((iterations % groups[j]) != 0) ? 1 : 0)); - cal_.iterations_ *= groups[j]; - groups[j] = 1; - } - } - } -} - -void VirtualGPU::setupIteration(uint iteration, const amd::NDRangeContainer& sizes, - Kernel& gpuKernel, amd::NDRange& global, amd::NDRange& offsets, - amd::NDRange& local, amd::NDRange& groups, - amd::NDRange& groupOffset, amd::NDRange& divider, - amd::NDRange& remainder, size_t extra) { - size_t dimensions = sizes.dimensions(); - - // Calculate the workload size for the remainder - if ((extra != 0) && ((iteration % extra) == 0)) { - groups = remainder; - } else { - groups = divider; - } - global = groups * local; - - for (uint j = 0; j < dimensions; ++j) { - size_t offset = groupOffset[j] * local[j]; - if ((offset + global[j]) > sizes.global()[j]) { - global[j] = sizes.global()[j] - offset; - } - } - - // Reprogram the kernel parameters for the GPU execution - gpuKernel.setupProgramGrid(*this, dimensions, offsets, global, local, groupOffset, sizes.offset(), - sizes.global()); - - // Update the constant buffers - gpuKernel.bindConstantBuffers(*this); - - uint sub = 0; - // Find the offsets for the next execution - for (uint j = 0; j < dimensions; ++j) { - groupOffset[j] += groups[j]; - // Make sure the offset doesn't go over the size limit - if (sizes.global()[j] <= groupOffset[j] * local[j]) { - // Check if we counted a group in one dimension already - if (sub) { - groupOffset[j] -= groups[j]; - } else { - groupOffset[j] = 0; - } - } else { - groupOffset[j] -= sub; - // We already counted elements in one dimension - sub = 1; - } - - offsets[j] = groupOffset[j] * local[j] + sizes.offset()[j]; - } -} - -void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd); - - // Submit kernel to HW - if (!submitKernelInternal(vcmd.sizes(), vcmd.kernel(), vcmd.parameters(), false, &vcmd.event())) { - vcmd.setStatus(CL_INVALID_OPERATION); - } - - profilingEnd(vcmd); -} - -bool VirtualGPU::submitKernelInternalHSA(const amd::NDRangeContainer& sizes, - const amd::Kernel& kernel, const_address parameters, - bool nativeMem, amd::Event* enqueueEvent) { - uint64_t vmParentWrap = 0; - uint64_t vmDefQueue = 0; - amd::DeviceQueue* defQueue = kernel.program().context().defDeviceQueue(dev()); - VirtualGPU* gpuDefQueue = NULL; - amd::HwDebugManager* dbgManager = dev().hwDebugMgr(); - - // Get the HSA kernel object - const HSAILKernel& hsaKernel = static_cast(*(kernel.getDeviceKernel(dev()))); - std::vector memList; - - bool printfEnabled = (hsaKernel.printfInfo().size() > 0) ? true : false; - if (!printfDbgHSA().init(*this, printfEnabled)) { - LogError("Printf debug buffer initialization failed!"); - return false; - } - - // Check memory dependency and SVM objects - if (!processMemObjectsHSA(kernel, parameters, nativeMem, &memList)) { - LogError("Wrong memory objects!"); - return false; - } - - cal_.memCount_ = 0; - - if (hsaKernel.dynamicParallelism()) { - if (NULL == defQueue) { - LogError("Default device queue wasn't allocated"); - return false; - } else { - if (dev().settings().useDeviceQueue_) { - gpuDefQueue = static_cast(defQueue->vDev()); - if (gpuDefQueue->hwRing() == hwRing()) { - LogError("Can't submit the child kernels to the same HW ring as the host queue!"); - return false; - } - } else { - createVirtualQueue(defQueue->size()); - gpuDefQueue = this; - } - } - vmDefQueue = gpuDefQueue->virtualQueue_->vmAddress(); - - // Add memory handles before the actual dispatch - memList.push_back(gpuDefQueue->virtualQueue_); - memList.push_back(gpuDefQueue->schedParams_); - memList.push_back(hsaKernel.prog().kernelTable()); - gpuDefQueue->writeVQueueHeader(*this, hsaKernel.prog().kernelTable()->vmAddress()); - } - - // setup the storage for the memory pointers of the kernel parameters - uint numParams = kernel.signature().numParameters(); - if (dbgManager) { - dbgManager->allocParamMemList(numParams); - } - - bool needFlush = false; - dmaFlushMgmt_.findSplitSize(dev(), sizes.global().product(), hsaKernel.aqlCodeSize()); - if (dmaFlushMgmt().dispatchSplitSize() != 0) { - needFlush = true; - } - - size_t newOffset[3] = {0, 0, 0}; - size_t newGlobalSize[3] = {0, 0, 0}; - - int dim = -1; - int iteration = 1; - size_t globalStep = 0; - for (uint i = 0; i < sizes.dimensions(); i++) { - newGlobalSize[i] = sizes.global()[i]; - newOffset[i] = sizes.offset()[i]; - } - // Check if it is blit kernel. If it is, then check if split is needed. - if (hsaKernel.isInternalKernel()) { - // Calculate new group size for each submission - for (uint i = 0; i < sizes.dimensions(); i++) { - if (sizes.global()[i] > static_cast(0xffffffff)) { - dim = i; - iteration = sizes.global()[i] / 0xC0000000 + ((sizes.global()[i] % 0xC0000000) ? 1 : 0); - globalStep = (sizes.global()[i] / sizes.local()[i]) / iteration * sizes.local()[dim]; - break; - } - } - } - - for (int j = 0; j < iteration; j++) { - // Reset global size for dimension dim if split is needed - if (dim != -1) { - newOffset[dim] = sizes.offset()[dim] + globalStep * j; - if (((newOffset[dim] + globalStep) < sizes.global()[dim]) && (j != (iteration - 1))) { - newGlobalSize[dim] = globalStep; - } else { - newGlobalSize[dim] = sizes.global()[dim] - newOffset[dim]; - } - } - - amd::NDRangeContainer tmpSizes(sizes.dimensions(), &newOffset[0], &newGlobalSize[0], - &(const_cast(sizes).local()[0])); - - // Program the kernel arguments for the GPU execution - hsa_kernel_dispatch_packet_t* aqlPkt = hsaKernel.loadArguments( - *this, kernel, tmpSizes, parameters, nativeMem, vmDefQueue, &vmParentWrap, memList); - if (NULL == aqlPkt) { - LogError("Couldn't load kernel arguments"); - return false; - } - - gslMemObject scratch = NULL; - uint scratchOffset = 0; - // Check if the device allocated more registers than the old setup - if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) { - const Device::ScratchBuffer* scratchObj = dev().scratch(hwRing()); - scratch = scratchObj->memObj_->gslResource(); - memList.push_back(scratchObj->memObj_); - scratchOffset = scratchObj->offset_; - } - - // Add GSL handle to the memory list for VidMM - for (uint i = 0; i < memList.size(); ++i) { - addVmMemory(memList[i]); - } - - // HW Debug for the kernel? - HwDbgKernelInfo kernelInfo; - HwDbgKernelInfo* pKernelInfo = NULL; - - if (dbgManager) { - buildKernelInfo(hsaKernel, aqlPkt, kernelInfo, enqueueEvent); - pKernelInfo = &kernelInfo; - } - - // Set up the dispatch information - KernelDispatchInfo dispatchInfo; - dispatchInfo.aqlPacket = aqlPkt; - dispatchInfo.mems = vmMems(); - dispatchInfo.numMems = cal_.memCount_; - dispatchInfo.scratch = scratch; - dispatchInfo.scratchOffset = scratchOffset; - dispatchInfo.cpuAqlCode = hsaKernel.cpuAqlCode(); - dispatchInfo.hsaQueueVA = hsaQueueMem_->vmAddress(); - dispatchInfo.kernelInfo = pKernelInfo; - dispatchInfo.wavesPerSH = hsaKernel.getWavesPerSH(this); - dispatchInfo.lastDoppSubmission = kernel.parameters().getExecNewVcop(); - dispatchInfo.pfpaDoppSubmission = kernel.parameters().getExecPfpaVcop(); - - GpuEvent gpuEvent; - // Run AQL dispatch in HW - eventBegin(MainEngine); - cs()->AqlDispatch(&dispatchInfo); - eventEnd(MainEngine, gpuEvent); - - if (dbgManager && (NULL != dbgManager->postDispatchCallBackFunc())) { - dbgManager->executePostDispatchCallBack(); - } - - if (hsaKernel.dynamicParallelism()) { - // Make sure exculsive access to the device queue - amd::ScopedLock(defQueue->lock()); - - if (GPU_PRINT_CHILD_KERNEL != 0) { - waitForEvent(&gpuEvent); - - AmdAqlWrap* wraps = - (AmdAqlWrap*)(&((AmdVQueueHeader*)gpuDefQueue->virtualQueue_->data())[1]); - uint p = 0; - for (uint i = 0; i < gpuDefQueue->vqHeader_->aql_slot_num; ++i) { - if (wraps[i].state != 0) { - uint j; - if (p == GPU_PRINT_CHILD_KERNEL) { - break; - } - p++; - std::stringstream print; - print.flags(std::ios::right | std::ios_base::hex | std::ios_base::uppercase); - print << "Slot#: " << i << "\n"; - print << "\tenqueue_flags: " << wraps[i].enqueue_flags << "\n"; - print << "\tcommand_id: " << wraps[i].command_id << "\n"; - print << "\tchild_counter: " << wraps[i].child_counter << "\n"; - print << "\tcompletion: " << wraps[i].completion << "\n"; - print << "\tparent_wrap: " << wraps[i].parent_wrap << "\n"; - print << "\twait_list: " << wraps[i].wait_list << "\n"; - print << "\twait_num: " << wraps[i].wait_num << "\n"; - uint offsEvents = wraps[i].wait_list - gpuDefQueue->virtualQueue_->vmAddress(); - size_t* events = - reinterpret_cast(gpuDefQueue->virtualQueue_->data() + offsEvents); - for (j = 0; j < wraps[i].wait_num; ++j) { - uint offs = - static_cast(events[j]) - gpuDefQueue->virtualQueue_->vmAddress(); - AmdEvent* eventD = (AmdEvent*)(gpuDefQueue->virtualQueue_->data() + offs); - print << "Wait Event#: " << j << "\n"; - print << "\tState: " << eventD->state << "; Counter: " << eventD->counter << "\n"; - } - print << "WorkGroupSize[ " << wraps[i].aql.workgroup_size_x << ", "; - print << wraps[i].aql.workgroup_size_y << ", "; - print << wraps[i].aql.workgroup_size_z << "]\n"; - print << "GridSize[ " << wraps[i].aql.grid_size_x << ", "; - print << wraps[i].aql.grid_size_y << ", "; - print << wraps[i].aql.grid_size_z << "]\n"; - - uint64_t* kernels = - (uint64_t*)(const_cast(hsaKernel.prog().kernelTable())->map(this)); - for (j = 0; j < hsaKernel.prog().kernels().size(); ++j) { - if (kernels[j] == wraps[i].aql.kernel_object) { - break; - } - } - const_cast(hsaKernel.prog().kernelTable())->unmap(this); - HSAILKernel* child = NULL; - for (auto it = hsaKernel.prog().kernels().begin(); - it != hsaKernel.prog().kernels().end(); ++it) { - if (j == static_cast(it->second)->index()) { - child = static_cast(it->second); - } - } - if (child == NULL) { - printf("Error: couldn't find child kernel!\n"); - continue; - } - const uint64_t kernarg_address = - static_cast(reinterpret_cast(wraps[i].aql.kernarg_address)); - uint offsArg = kernarg_address - gpuDefQueue->virtualQueue_->vmAddress(); - address argum = gpuDefQueue->virtualQueue_->data() + offsArg; - print << "Kernel: " << child->name() << "\n"; - static const char* Names[HSAILKernel::MaxExtraArgumentsNum] = { - "Offset0: ", "Offset1: ", "Offset2: ", "PrintfBuf: ", "VqueuePtr: ", "AqlWrap: "}; - for (j = 0; j < child->extraArgumentsNum(); ++j) { - print << "\t" << Names[j] << *(size_t*)argum; - print << "\n"; - argum += sizeof(size_t); - } - for (j = 0; j < child->numArguments(); ++j) { - print << "\t" << child->argument(j)->name_ << ": "; - for (int s = child->argument(j)->size_ - 1; s >= 0; --s) { - print.width(2); - print.fill('0'); - print << (uint32_t)(argum[s]); - } - argum += child->argument(j)->size_; - print << "\n"; - } - printf("%s", print.str().c_str()); - } - } - } - - if (!dev().settings().useDeviceQueue_) { - // Add the termination handshake to the host queue - eventBegin(MainEngine); - cs()->VirtualQueueHandshake(gpuDefQueue->schedParams_->gslResource(), - vmParentWrap + offsetof(AmdAqlWrap, state), AQL_WRAP_DONE, - vmParentWrap + offsetof(AmdAqlWrap, child_counter), 0, - dev().settings().useDeviceQueue_); - eventEnd(MainEngine, gpuEvent); - } - - // Get the global loop start before the scheduler - mcaddr loopStart = gpuDefQueue->cs()->VirtualQueueDispatcherStart(); - static_cast(gpuDefQueue->blitMgr()) - .runScheduler(*gpuDefQueue->virtualQueue_, *gpuDefQueue->schedParams_, - gpuDefQueue->schedParamIdx_, - gpuDefQueue->vqHeader_->aql_slot_num / (DeviceQueueMaskSize * maskGroups_)); - const static bool FlushL2 = true; - gpuDefQueue->flushCUCaches(FlushL2); - - // Get the address of PM4 template and add write it to params - //! @note DMA flush must not occur between patch and the scheduler - mcaddr patchStart = gpuDefQueue->cs()->VirtualQueueDispatcherStart(); - - // Program parameters for the scheduler - SchedulerParam* param = &reinterpret_cast( - gpuDefQueue->schedParams_->data())[gpuDefQueue->schedParamIdx_]; - param->signal = 1; - // Scale clock to 1024 to avoid 64 bit div in the scheduler - param->eng_clk = (1000 * 1024) / dev().info().maxEngineClockFrequency_; - param->hw_queue = patchStart + sizeof(uint32_t) /* Rewind packet*/; - param->hsa_queue = gpuDefQueue->hsaQueueMem()->vmAddress(); - param->releaseHostCP = 0; - param->parentAQL = vmParentWrap; - param->dedicatedQueue = dev().settings().useDeviceQueue_; - param->useATC = dev().settings().svmFineGrainSystem_; - - // Fill the scratch buffer information - if (hsaKernel.prog().maxScratchRegs() > 0) { - gpu::Memory* scratchBuf = dev().scratch(gpuDefQueue->hwRing())->memObj_; - param->scratchSize = scratchBuf->size(); - param->scratch = scratchBuf->vmAddress(); - param->numMaxWaves = 32 * dev().info().maxComputeUnits_; - param->scratchOffset = dev().scratch(gpuDefQueue->hwRing())->offset_; - memList.push_back(scratchBuf); - } else { - param->numMaxWaves = 0; - param->scratchSize = 0; - param->scratch = 0; - param->scratchOffset = 0; - } - - // Add all kernels in the program to the mem list. - //! \note Runtime doesn't know which one will be called - hsaKernel.prog().fillResListWithKernels(memList); - - // Add GSL handle to the memory list for VidMM - for (uint i = 0; i < memList.size(); ++i) { - gpuDefQueue->addVmMemory(memList[i]); - } - - mcaddr signalAddr = gpuDefQueue->schedParams_->vmAddress() + - gpuDefQueue->schedParamIdx_ * sizeof(SchedulerParam); - gpuDefQueue->eventBegin(MainEngine); - gpuDefQueue->cs()->VirtualQueueDispatcherEnd( - gpuDefQueue->vmMems(), gpuDefQueue->cal_.memCount_, signalAddr, loopStart, - gpuDefQueue->vqHeader_->aql_slot_num / (DeviceQueueMaskSize * maskGroups_)); - gpuDefQueue->eventEnd(MainEngine, gpuEvent); - - // Set GPU event for the used resources - for (uint i = 0; i < memList.size(); ++i) { - memList[i]->setBusy(*gpuDefQueue, gpuEvent); - } - - if (dev().settings().useDeviceQueue_) { - // Add the termination handshake to the host queue - eventBegin(MainEngine); - cs()->VirtualQueueHandshake(gpuDefQueue->schedParams_->gslResource(), - vmParentWrap + offsetof(AmdAqlWrap, state), AQL_WRAP_DONE, - vmParentWrap + offsetof(AmdAqlWrap, child_counter), signalAddr, - dev().settings().useDeviceQueue_); - eventEnd(MainEngine, gpuEvent); - } - - ++gpuDefQueue->schedParamIdx_ %= gpuDefQueue->schedParams_->size() / sizeof(SchedulerParam); - //! \todo optimize the wrap around - if (gpuDefQueue->schedParamIdx_ == 0) { - gpuDefQueue->schedParams_->wait(*gpuDefQueue); - } - } - - // Set GPU event for the used resources - for (uint i = 0; i < memList.size(); ++i) { - memList[i]->setBusy(*this, gpuEvent); - } - - // Update the global GPU event - setGpuEvent(gpuEvent, needFlush); - - if (!printfDbgHSA().output(*this, printfEnabled, hsaKernel.printfInfo())) { - LogError("Couldn't read printf data from the buffer!\n"); - return false; - } - } - - // Runtime submitted a HSAIL kernel - state_.hsailKernel_ = true; - - return true; -} - -bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const amd::Kernel& kernel, - const_address parameters, bool nativeMem, - amd::Event* enqueueEvent) { - bool result = true; - uint i; - size_t dimensions = sizes.dimensions(); - amd::NDRange local(sizes.local()); - amd::NDRange groupOffset(dimensions); - GpuEvent gpuEvent; - groupOffset = 0; - - // Get the GPU kernel object - device::Kernel* devKernel = const_cast(kernel.getDeviceKernel(dev())); - Kernel& gpuKernelOpt = static_cast(*devKernel); - - if (gpuKernelOpt.hsa()) { - return submitKernelInternalHSA(sizes, kernel, parameters, nativeMem, enqueueEvent); - } else if (state_.hsailKernel_) { - // Reload GSL state to HW, so runtime could run AMDIL kernel - flushDMA(MainEngine); - // Reset HSAIL state - state_.hsailKernel_ = false; - } - - // Find if arguments contain memory aliases or a dependency in the queue - gpuKernelOpt.processMemObjects(*this, kernel, parameters, nativeMem); - - Kernel& gpuKernel = static_cast(*devKernel); - bool printfEnabled = (gpuKernel.flags() & gpu::NullKernel::PrintfOutput) ? true : false; - // Set current kernel CAL descriptor as active - if (!setActiveKernelDesc(sizes, &gpuKernel) || - // Initialize printf support - !printfDbg().init(*this, printfEnabled, sizes.global())) { - LogPrintfError("We couldn't set \"%s\" kernel as active!", gpuKernel.name().data()); - return false; - } - - // Find if we have to split workload - dmaFlushMgmt_.findSplitSize(dev(), sizes.global().product(), gpuKernel.instructionCnt()); - - // Program the kernel parameters for the GPU execution - cal_.memCount_ = 0; - gpuKernel.setupProgramGrid(*this, dimensions, sizes.offset(), sizes.global(), local, groupOffset, - sizes.offset(), sizes.global()); - - // Load kernel arguments - if (gpuKernel.loadParameters(*this, kernel, parameters, nativeMem)) { - amd::NDRange global(sizes.global()); - amd::NDRange groups(dimensions); - amd::NDRange offsets(sizes.offset()); - amd::NDRange divider(dimensions); - amd::NDRange remainder(dimensions); - size_t extra = 0; - - // Split the workload if necessary for local/private emulation or printf - findIterations(sizes, local, groups, remainder, extra); - - divider = groups; - i = 0; - do { - bool lastRun = (i == (cal()->iterations_ - 1)) ? true : false; - // Reprogram the CAL grid and constant buffers if - // the workload split is on - if (cal()->iterations_ > 1) { - // Initialize printf support - if (!printfDbg().init(*this, printfEnabled, local)) { - result = false; - break; - } - - // Reprogram the CAL grid and constant buffers - setupIteration(i, sizes, gpuKernel, global, offsets, local, groups, groupOffset, divider, - remainder, extra); - } - - // Execute the kernel - if (gpuKernel.run(*this, &gpuEvent, lastRun, kernel.parameters().getExecNewVcop(), - kernel.parameters().getExecPfpaVcop())) { - //! @todo A flush is necessary to make sure - // that 2 consecutive runs won't access to the same - // private/local memory. CAL has to generate cache flush - // and wait for idle commands - bool flush = ((cal()->iterations_ > 1) || - dmaFlushMgmt_.isCbReady(*this, global.product(), gpuKernel.instructionCnt())) - ? true - : false; - - // Update the global GPU event - setGpuEvent(gpuEvent, flush); - - // This code for the kernel execution debugging - if (dev().settings().debugFlags_ & Settings::LockGlobalMemory) { - gpuKernel.debug(*this); - } - } else { - result = false; - break; - } - - // Print the debug buffer output result - if (printfDbg().output(*this, printfEnabled, - (cal()->iterations_ > 1) ? local : sizes.global(), - gpuKernel.prog().printfInfo())) { - // Go to the next iteration - ++i; - } else { - result = false; - break; - } - } - // Check if we have to make multiple iterations - while (i < cal()->iterations_); - } else { - result = false; - } - - if (!result) { - LogPrintfError("submitKernel failed to execute the \"%s\" kernel on HW!", - gpuKernel.name().data()); - } - - return result; -} - -void VirtualGPU::submitNativeFn(amd::NativeFnCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - Unimplemented(); //!< @todo: Unimplemented -} - -void VirtualGPU::submitMarker(amd::Marker& vcmd) { - //!@note runtime doesn't need to lock this command on execution - - if (vcmd.waitingEvent() != NULL) { - bool foundEvent = false; - - // Loop through all outstanding command batches - while (!cbList_.empty()) { - const auto it = cbList_.cbegin(); - // Wait for completion - foundEvent = awaitCompletion(*it, vcmd.waitingEvent()); - // Release a command batch - delete *it; - // Remove command batch from the list - cbList_.pop_front(); - // Early exit if we found a command - if (foundEvent) break; - } - - // Event should be in the current command batch - if (!foundEvent) { - state_.forceWait_ = true; - } - // If we don't have any more batches, then assume GPU is idle - else if (cbList_.empty()) { - dmaFlushMgmt_.resetCbWorkload(dev()); - } - } -} - -void VirtualGPU::releaseMemory(gslMemObject gslResource, bool wait) { - bool result = true; - if (wait) { - waitForEvent(&gpuEvents_[gslResource]); - } - - // Unbind resource if it's active kernel desc - for (uint i = 0; i < MaxUavArguments; ++i) { - if (gslResource == cal_.uavs_[i]) { - result = setUAVBuffer(i, 0, GSL_UAV_TYPE_UNKNOWN); - cal_.uavs_[i] = 0; - } - } - for (uint i = 0; i < MaxReadImage; ++i) { - if (gslResource == cal_.readImages_[i]) { - result = setInput(i, 0); - cal_.readImages_[i] = 0; - } - } - for (uint i = 0; i < MaxConstBuffers; ++i) { - if (gslResource == cal_.constBuffers_[i]) { - result = setConstantBuffer(i, 0, 0, 0); - cal_.constBuffers_[i] = 0; - } - } - - if ((dev().scratch(hwRing()) != NULL) && (dev().scratch(hwRing())->regNum_ > 0)) { - // Unbind scratch memory - const Device::ScratchBuffer* scratch = dev().scratch(hwRing()); - if ((scratch->memObj_ != NULL) && (scratch->memObj_->gslResource() == gslResource)) { - setScratchBuffer(NULL, 0); - } - } - - gpuEvents_.erase(gslResource); -} - -void VirtualGPU::releaseKernel(CALimage calImage) { - GslKernelDesc* desc = gslKernels_[calImage]; - if (desc != NULL) { - freeKernelDesc(desc); - } - gslKernels_.erase(calImage); -} - -void VirtualGPU::submitPerfCounter(amd::PerfCounterCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - gslQueryObject gslCounter; - - const amd::PerfCounterCommand::PerfCounterList counters = vcmd.getCounters(); - - // Create a HW counter - gslCounter = cs()->createQuery(GSL_PERFORMANCE_COUNTERS_ATI); - if (0 == gslCounter) { - LogError("We failed to allocate memory for the GPU perfcounter"); - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - CalCounterReference* calRef = new CalCounterReference(*this, gslCounter); - if (calRef == NULL) { - LogError("We failed to allocate memory for the GPU perfcounter"); - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - gslCounter = 0; - - for (uint i = 0; i < vcmd.getNumCounters(); ++i) { - amd::PerfCounter* amdCounter = static_cast(counters[i]); - const PerfCounter* counter = static_cast(amdCounter->getDeviceCounter()); - - // Make sure we have a valid gpu performance counter - if (NULL == counter) { - amd::PerfCounter::Properties prop = amdCounter->properties(); - PerfCounter* gpuCounter = new PerfCounter( - gpuDevice_, *this, prop[CL_PERFCOUNTER_GPU_BLOCK_INDEX], - prop[CL_PERFCOUNTER_GPU_COUNTER_INDEX], prop[CL_PERFCOUNTER_GPU_EVENT_INDEX]); - if (NULL == gpuCounter) { - LogError("We failed to allocate memory for the GPU perfcounter"); - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } else if (gpuCounter->create(calRef)) { - amdCounter->setDeviceCounter(gpuCounter); - } else { - LogPrintfError( - "We failed to allocate a perfcounter in CAL.\ - Block: %d, counter: #d, event: %d", - gpuCounter->info()->blockIndex_, gpuCounter->info()->counterIndex_, - gpuCounter->info()->eventIndex_); - delete gpuCounter; - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - counter = gpuCounter; - } - } - - calRef->release(); - - for (uint i = 0; i < vcmd.getNumCounters(); ++i) { - amd::PerfCounter* amdCounter = static_cast(counters[i]); - const PerfCounter* counter = static_cast(amdCounter->getDeviceCounter()); - - if (gslCounter != counter->gslCounter()) { - gslCounter = counter->gslCounter(); - // Find the state and sends the command to CAL - if (vcmd.getState() == amd::PerfCounterCommand::Begin) { - gslCounter->BeginQuery(cs(), GSL_PERFORMANCE_COUNTERS_ATI, 0); - } else if (vcmd.getState() == amd::PerfCounterCommand::End) { - GpuEvent event; - eventBegin(MainEngine); - gslCounter->EndQuery(cs(), 0); - eventEnd(MainEngine, event); - setGpuEvent(event); - } else { - LogError("Unsupported performance counter state"); - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - } - } -} -void VirtualGPU::submitThreadTraceMemObjects(amd::ThreadTraceMemObjectsCommand& cmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(cmd); - - switch (cmd.type()) { - case CL_COMMAND_THREAD_TRACE_MEM: { - amd::ThreadTrace* amdThreadTrace = &cmd.getThreadTrace(); - ThreadTrace* threadTrace = static_cast(amdThreadTrace->getDeviceThreadTrace()); - - if (threadTrace == NULL) { - gslQueryObject gslThreadTrace; - // Create a HW thread trace query object - gslThreadTrace = cs()->createQuery(GSL_SHADER_TRACE_BYTES_WRITTEN); - if (0 == gslThreadTrace) { - LogError("Failure in memory allocation for the GPU threadtrace"); - cmd.setStatus(CL_INVALID_OPERATION); - return; - } - CalThreadTraceReference* calRef = new CalThreadTraceReference(*this, gslThreadTrace); - if (calRef == NULL) { - LogError("Failure in memory allocation for the GPU threadtrace"); - cmd.setStatus(CL_INVALID_OPERATION); - return; - } - size_t seNum = amdThreadTrace->deviceSeNumThreadTrace(); - ThreadTrace* gpuThreadTrace = new ThreadTrace(gpuDevice_, *this, seNum); - if (NULL == gpuThreadTrace) { - LogError("Failure in memory allocation for the GPU threadtrace"); - cmd.setStatus(CL_INVALID_OPERATION); - return; - } - if (gpuThreadTrace->create(calRef)) { - amdThreadTrace->setDeviceThreadTrace(gpuThreadTrace); - } else { - LogError("Failure in memory allocation for the GPU threadtrace"); - delete gpuThreadTrace; - cmd.setStatus(CL_INVALID_OPERATION); - return; - } - threadTrace = gpuThreadTrace; - calRef->release(); - } - gslShaderTraceBufferObject* threadTraceBufferObjects = - threadTrace->getThreadTraceBufferObjects(); - const size_t memObjSize = cmd.getMemoryObjectSize(); - const std::vector& memObj = cmd.getMemList(); - size_t se = 0; - for (auto itMemObj = memObj.cbegin(); - itMemObj != memObj.cend(); ++itMemObj, ++se) { - // Find GSL Mem Object - gslMemObject gslMemObj = dev().getGpuMemory(*itMemObj)->gslResource(); - - // Bind GSL MemObject to the appropriate SE Thread Trace Buffer Object - threadTraceBufferObjects[se]->attachMemObject(cs(), gslMemObj, 0, 0, memObjSize, se); - } - break; - } - default: - LogError("Unsupported command type for ThreadTraceMemObjects!"); - break; - } -} - -void VirtualGPU::submitThreadTrace(amd::ThreadTraceCommand& cmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(cmd); - - switch (cmd.type()) { - case CL_COMMAND_THREAD_TRACE: { - amd::ThreadTrace* amdThreadTrace = static_cast(&cmd.getThreadTrace()); - ThreadTrace* threadTrace = static_cast(amdThreadTrace->getDeviceThreadTrace()); - - // gpu thread trace object had to be generated prior to begin/end/pause/resume due - // to ThreadTraceMemObjectsCommand execution - if (threadTrace == NULL) { - return; - } else { - gslQueryObject gslThreadTrace; - gslThreadTrace = threadTrace->gslThreadTrace(); - uint32_t seNum = amdThreadTrace->deviceSeNumThreadTrace(); - - // Find the state and sends the commands to GSL - if (cmd.getState() == amd::ThreadTraceCommand::Begin) { - amd::ThreadTrace::ThreadTraceConfig* traceCfg = - static_cast(cmd.threadTraceConfig()); - const gslErrorCode ec = - gslThreadTrace->BeginQuery(cs(), GSL_SHADER_TRACE_BYTES_WRITTEN, 0); - assert(ec == GSL_NO_ERROR); - - for (uint32_t idx = 0; idx < seNum; ++idx) { - rs()->enableShaderTrace(cs(), idx, true); - rs()->setShaderTraceComputeUnit(idx, traceCfg->cu_); - rs()->setShaderTraceShaderArray(idx, traceCfg->sh_); - rs()->setShaderTraceSIMDMask(idx, traceCfg->simdMask_); - rs()->setShaderTraceVmIdMask(idx, traceCfg->vmIdMask_); - rs()->setShaderTraceTokenMask(idx, traceCfg->tokenMask_); - rs()->setShaderTraceRegisterMask(idx, traceCfg->regMask_); - rs()->setShaderTraceIssueMask(idx, traceCfg->instMask_); - rs()->setShaderTraceRandomSeed(idx, traceCfg->randomSeed_); - rs()->setShaderTraceCaptureMode(idx, traceCfg->captureMode_); - rs()->setShaderTraceWrap(idx, traceCfg->isWrapped_); - rs()->setShaderTraceUserData(idx, (traceCfg->isUserData_) ? traceCfg->userData_ : 0); - } - } else if (cmd.getState() == amd::ThreadTraceCommand::End) { - for (uint32_t idx = 0; idx < seNum; ++idx) { - rs()->enableShaderTrace(cs(), idx, false); - } - gslThreadTrace->EndQuery(cs(), 0); - } else if (cmd.getState() == amd::ThreadTraceCommand::Pause) { - for (uint32_t idx = 0; idx < seNum; ++idx) { - rs()->setShaderTraceIsPaused(cs(), idx, true); - } - } else if (cmd.getState() == amd::ThreadTraceCommand::Resume) { - for (uint32_t idx = 0; idx < seNum; ++idx) { - rs()->setShaderTraceIsPaused(cs(), idx, false); - } - } - } - break; - } - default: - LogError("Unsupported command type for ThreadTrace!"); - break; - } -} - -void VirtualGPU::submitAcquireExtObjects(amd::AcquireExtObjectsCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd); - - for (const auto& it : vcmd.getMemList()) { - // amd::Memory object should never be NULL - assert(it && "Memory object for interop is NULL"); - gpu::Memory* memory = dev().getGpuMemory(it); - - // If resource is a shared copy of original resource, then - // runtime needs to copy data from original resource - it->getInteropObj()->copyOrigToShared(); - - // Check if OpenCL has direct access to the interop memory - if (memory->interopType() == Memory::InteropDirectAccess) { - continue; - } - - // Does interop use HW emulation? - if (memory->interopType() == Memory::InteropHwEmulation) { - static const bool Entire = true; - amd::Coord3D origin(0, 0, 0); - amd::Coord3D region(memory->size()); - - // Synchronize the object - if (!blitMgr().copyBuffer(*memory->interop(), *memory, origin, origin, region, Entire)) { - LogError("submitAcquireExtObjects - Interop synchronization failed!"); - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - } - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitReleaseExtObjects(amd::ReleaseExtObjectsCommand& vcmd) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - profilingBegin(vcmd); - - for (const auto& it : vcmd.getMemList()) { - // amd::Memory object should never be NULL - assert(it && "Memory object for interop is NULL"); - gpu::Memory* memory = dev().getGpuMemory(it); - - // Check if we can use HW interop - if (memory->interopType() == Memory::InteropHwEmulation) { - static const bool Entire = true; - amd::Coord3D origin(0, 0, 0); - amd::Coord3D region(memory->size()); - - // Synchronize the object - if (!blitMgr().copyBuffer(*memory, *memory->interop(), origin, origin, region, Entire)) { - LogError("submitReleaseExtObjects interop synchronization failed!"); - vcmd.setStatus(CL_INVALID_OPERATION); - return; - } - } else { - if (memory->interopType() != Memory::InteropDirectAccess) { - LogError("None interop release!"); - } - } - - // If resource is a shared copy of original resource, then - // runtime needs to copy data back to original resource - it->getInteropObj()->copySharedToOrig(); - } - - profilingEnd(vcmd); -} - -void VirtualGPU::submitSignal(amd::SignalCommand& vcmd) { - amd::ScopedLock lock(execution()); - profilingBegin(vcmd); - gpu::Memory* gpuMemory = dev().getGpuMemory(&vcmd.memory()); - GpuEvent gpuEvent; - eventBegin(MainEngine); - if (vcmd.type() == CL_COMMAND_WAIT_SIGNAL_AMD) { - uint64_t surfAddr = gpuMemory->gslResource()->getPhysicalAddress(cs()); - uint64_t markerAddr = gpuMemory->gslResource()->getMarkerAddress(cs()); - uint64_t markerOffset = markerAddr - surfAddr; - cs()->p2pMarkerOp(gpuMemory->gslResource(), vcmd.markerValue(), markerOffset, false); - } else if (vcmd.type() == CL_COMMAND_WRITE_SIGNAL_AMD) { - static constexpr bool FlushL2 = true; - flushCUCaches(FlushL2); - cs()->p2pMarkerOp(gpuMemory->gslResource(), vcmd.markerValue(), vcmd.markerOffset(), true); - } - eventEnd(MainEngine, gpuEvent); - gpuMemory->setBusy(*this, gpuEvent); - // Update the global GPU event - setGpuEvent(gpuEvent); - - profilingEnd(vcmd); -} - -void VirtualGPU::submitMakeBuffersResident(amd::MakeBuffersResidentCommand& vcmd) { - amd::ScopedLock lock(execution()); - profilingBegin(vcmd); - std::vector memObjects = vcmd.memObjects(); - uint32_t numObjects = memObjects.size(); - gslMemObject* pGSLMemObjects = new gslMemObject[numObjects]; - - for (uint32_t i = 0; i < numObjects; ++i) { - gpu::Memory* gpuMemory = dev().getGpuMemory(memObjects[i]); - pGSLMemObjects[i] = gpuMemory->gslResource(); - gpuMemory->syncCacheFromHost(*this); - } - - uint64* surfBusAddr = new uint64[numObjects]; - uint64* markerBusAddr = new uint64[numObjects]; - gslErrorCode res = - cs()->makeBuffersResident(numObjects, pGSLMemObjects, surfBusAddr, markerBusAddr); - if (res != GSL_NO_ERROR) { - LogError("MakeBuffersResident failed"); - vcmd.setStatus(CL_INVALID_OPERATION); - } else { - cl_bus_address_amd* busAddr = vcmd.busAddress(); - for (uint32_t i = 0; i < numObjects; ++i) { - busAddr[i].surface_bus_address = surfBusAddr[i]; - busAddr[i].marker_bus_address = markerBusAddr[i]; - } - } - delete[] pGSLMemObjects; - delete[] surfBusAddr; - delete[] markerBusAddr; - profilingEnd(vcmd); -} - - -bool VirtualGPU::awaitCompletion(CommandBatch* cb, const amd::Event* waitingEvent) { - bool found = false; - amd::Command* current; - amd::Command* head = cb->head_; - - // Make sure that profiling is enabled - if (profileEnabled_) { - return profilingCollectResults(cb, waitingEvent); - } - // Mark the first command in the batch as running - if (head != NULL) { - head->setStatus(CL_RUNNING); - } else { - return found; - } - - // Wait for the last known GPU event - waitEventLock(cb); - - while (NULL != head) { - current = head->getNext(); - if (head->status() == CL_SUBMITTED) { - head->setStatus(CL_RUNNING); - head->setStatus(CL_COMPLETE); - } else if (head->status() == CL_RUNNING) { - head->setStatus(CL_COMPLETE); - } else if ((head->status() != CL_COMPLETE) && (current != NULL)) { - LogPrintfError("Unexpected command status - %d!", head->status()); - } - - // Check if it's a waiting command - if (head == waitingEvent) { - found = true; - } - - head->release(); - head = current; - } - - return found; -} - -void VirtualGPU::flush(amd::Command* list, bool wait) { - CommandBatch* cb = NULL; - bool gpuCommand = false; - - for (uint i = 0; i < AllEngines; ++i) { - if (cal_.events_[i].isValid()) { - gpuCommand = true; - } - } - - // If the batch doesn't have any GPU command and the list is empty - if (!gpuCommand && cbList_.empty()) { - state_.forceWait_ = true; - } - - // Insert the current batch into a list - if (NULL != list) { - cb = new CommandBatch(list, cal()->events_, cal()->lastTS_); - } - - { - //! @note: flushDMA() requires a lock, because GSL can - //! defer destruction of internal memory objects and releases them - //! on GSL flush. If runtime calls another GSL flush at the same time, - //! then double release can occur. - amd::ScopedLock lock(execution()); - for (uint i = 0; i < AllEngines; ++i) { - flushDMA(i); - // Reset event so we won't try to wait again, - // if runtime didn't submit any commands - //! @note: it's safe to invalidate events, since - //! we already saved them with the batch creation step above - cal_.events_[i].invalidate(); - } - } - - // Mark last TS as NULL, so runtime won't process empty batches with the old TS - cal_.lastTS_ = NULL; - if (NULL != cb) { - cbList_.push_back(cb); - } - - wait |= state_.forceWait_; - // Loop through all outstanding command batches - while (!cbList_.empty()) { - const auto it = cbList_.cbegin(); - // Check if command batch finished without a wait - bool finished = true; - for (uint i = 0; i < AllEngines; ++i) { - finished &= isDone(&(*it)->events_[i]); - } - if (finished || wait) { - // Wait for completion - awaitCompletion(*it); - // Release a command batch - delete *it; - // Remove command batch from the list - cbList_.pop_front(); - } else { - // Early exit if no finished - break; - } - } - state_.forceWait_ = false; -} - -void VirtualGPU::enableSyncedBlit() const { return blitMgr_->enableSynchronization(); } - -void VirtualGPU::releaseMemObjects(bool scratch) { - for (const auto& it : gpuEvents_) { - GpuEvent event = it.second; - waitForEvent(&event); - } - // Unbind all resources.So the queue won't have any bound mem objects - for (uint i = 0; i < MaxUavArguments; ++i) { - if (NULL != cal_.uavs_[i]) { - setUAVBuffer(i, 0, GSL_UAV_TYPE_UNKNOWN); - cal_.uavs_[i] = 0; - } - } - for (uint i = 0; i < MaxReadImage; ++i) { - if (NULL != cal_.readImages_[i]) { - setInput(i, 0); - cal_.readImages_[i] = 0; - } - } - for (uint i = 0; i < MaxConstBuffers; ++i) { - if (NULL != cal_.constBuffers_[i]) { - setConstantBuffer(i, 0, 0, 0); - cal_.constBuffers_[i] = 0; - } - } - - if (scratch) { - setScratchBuffer(NULL, 0); - } - - gpuEvents_.clear(); -} - -void VirtualGPU::setGpuEvent(GpuEvent gpuEvent, bool flush) { - cal_.events_[engineID_] = gpuEvent; - - // Flush current DMA buffer if requested - if (flush || GPU_FLUSH_ON_EXECUTION) { - flushDMA(engineID_); - } -} - -void VirtualGPU::flushDMA(uint engineID) { - if (engineID == MainEngine) { - // Clear memory dependency state, since runtime flushes compute - // memoryDependency().clear(); - //!@todo Keep memory dependency alive even if we flush DMA, - //! since only L2 cache is flushed in KMD frame, - //! but L1 still has to be invalidated. - } - //! \note Use CtxIsEventDone, so we won't flush compute for DRM engine - isDone(&cal_.events_[engineID]); -} - -bool VirtualGPU::waitAllEngines(CommandBatch* cb) { - uint i; - GpuEvent* events; //!< GPU events for the batch - // If command batch is NULL then wait for the current - if (NULL == cb) { - events = cal_.events_; - } else { - events = cb->events_; - } - - bool earlyDone = true; - // The first loop is to flush all engines and/or check if - // engines are idle already - for (i = 0; i < AllEngines; ++i) { - earlyDone &= isDone(&events[i]); - } - - // Release all transfer buffers on this command queue - releaseXferWrite(); - - // Rlease all pinned memory - releasePinnedMem(); - - // The second loop is to wait all engines - for (i = 0; i < AllEngines; ++i) { - waitForEvent(&events[i]); - } - - return earlyDone; -} - -void VirtualGPU::waitEventLock(CommandBatch* cb) { - // Make sure VirtualGPU has an exclusive access to the resources - amd::ScopedLock lock(execution()); - - bool earlyDone = waitAllEngines(cb); - - // Free resource cache if we have too many entries - //! \note we do it here, when all engines are idle, - // because Vista/Win7 idles GPU on a resource destruction - static const size_t MinCacheEntries = 4096; - dev().resourceCache().free(MinCacheEntries); - - // Find the timestamp object of the last command in the batch - if (cb->lastTS_ != NULL) { - // If earlyDone is TRUE, then CPU didn't wait for GPU. - // Thus the sync point between CPU and GPU is unclear and runtime - // will use an older adjustment value to maintain the same timeline - if (!earlyDone || - //! \note Workaround for APU(s). - //! GPU-CPU timelines may go off too much, thus always - //! force calibration with the last batch in the list - (cbList_.size() <= 1) || (readjustTimeGPU_ == 0)) { - uint64_t startTimeStampGPU = 0; - uint64_t endTimeStampGPU = 0; - - // Get the timestamp value of the last command in the batch - cb->lastTS_->value(&startTimeStampGPU, &endTimeStampGPU); - - uint64_t endTimeStampCPU = amd::Os::timeNanos(); - // Make sure the command batch has a valid GPU TS - if (!GPU_RAW_TIMESTAMP) { - // Adjust the base time by the execution time - readjustTimeGPU_ = endTimeStampGPU - endTimeStampCPU; - } - } - } -} - -void VirtualGPU::validateScratchBuffer(const Kernel* kernel) { - // Check if a scratch buffer is required - if (dev().scratch(hwRing())->regNum_ > 0) { - // Setup scratch buffer - setScratchBuffer(dev().scratch(hwRing())->memObj_->gslResource(), 0); - } -} - -bool VirtualGPU::setActiveKernelDesc(const amd::NDRangeContainer& sizes, const Kernel* kernel) { - bool result = true; - CALimage calImage = kernel->calImage(); - - GslKernelDesc* desc = gslKernels_[calImage]; - - validateScratchBuffer(kernel); - - // Early exit - if ((activeKernelDesc_ == desc) && (desc != NULL)) { - return result; - } - - // Does the kernel descriptor for this virtual device exist? - if (desc == NULL) { - desc = allocKernelDesc(kernel, calImage); - if (desc == NULL) { - return false; - } - gslKernels_[calImage] = desc; - } - - // Set the descriptor as active - activeKernelDesc_ = desc; - - // Program the samplers defined in the kernel - if (!kernel->setInternalSamplers(*this)) { - result = false; - } - - // Bind global HW constant buffers - if (!kernel->bindGlobalHwCb(*this, desc)) { - result = false; - } - - if (result) { - // Set program in GSL - rs()->setCurrentProgramObject(GSL_COMPUTE_PROGRAM, desc->func_); - - // Update internal constant buffer - if (desc->intCb_ != 0) { - cs()->setIntConstants(GSL_COMPUTE_PROGRAM, desc->intCb_); - } - } - - return result; -} - -bool VirtualGPU::allocConstantBuffers() { - // Allocate/reallocate constant buffers - size_t minCbSize; - // GCN doesn't really have a limit - minCbSize = 128 * Ki; - uint i; - - // Create/reallocate constant buffer resources - for (i = 0; i < MaxConstBuffersArguments; ++i) { - ConstBuffer* constBuf = new ConstBuffer( - *this, ((minCbSize + ConstBuffer::VectorSize - 1) / ConstBuffer::VectorSize)); - - if ((constBuf != NULL) && constBuf->create()) { - addConstBuffer(constBuf); - } else { - // We failed to create a constant buffer - delete constBuf; - return false; - } - } - - return true; -} - -VirtualGPU::GslKernelDesc* VirtualGPU::allocKernelDesc(const Kernel* kernel, CALimage calImage) { - // Sanity checks - assert(kernel != NULL); - GslKernelDesc* desc = new GslKernelDesc; - - if (desc != NULL) { - memset(desc, 0, sizeof(GslKernelDesc)); - - if (kernel->calImage() != calImage) { - desc->image_ = calImage; - } - - if (!moduleLoad(calImage, &desc->func_, &desc->intCb_)) { - LogPrintfError("calModuleLoad failed for \"%s\" kernel!", kernel->name().c_str()); - delete desc; - return NULL; - } - } - - if (kernel->argSize() > slots_.size()) { - slots_.resize(kernel->argSize()); - } - - return desc; -} - -void VirtualGPU::freeKernelDesc(VirtualGPU::GslKernelDesc* desc) { - if (desc) { - if (gslKernelDesc() == desc) { - // Clear active kernel desc - activeKernelDesc_ = NULL; - rs()->setCurrentProgramObject(GSL_COMPUTE_PROGRAM, 0); - } - - if (desc->image_ != 0) { - // Free CAL image - free(desc->image_); - } - - if (desc->func_ != 0) { - if (desc->intCb_ != 0) { - cs()->setIntConstants(GSL_COMPUTE_PROGRAM, 0); - cs()->destroyMemObject(desc->intCb_); - } - cs()->destroyProgramObject(desc->func_); - } - - delete desc; - } -} - -void VirtualGPU::profilingBegin(amd::Command& command, bool drmProfiling) { - // Is profiling enabled? - if (command.profilingInfo().enabled_) { - // Allocate a timestamp object from the cache - TimeStamp* ts = tsCache_->allocTimeStamp(); - if (NULL == ts) { - return; - } - // Save the TimeStamp object in the current OCL event - command.setData(ts); - currTs_ = ts; - profileEnabled_ = true; - } -} - -void VirtualGPU::profilingEnd(amd::Command& command) { - // Get the TimeStamp object associated witht the current command - TimeStamp* ts = reinterpret_cast(command.data()); - if (ts != NULL) { - // Check if the command actually did any GPU submission - if (ts->isValid()) { - cal_.lastTS_ = ts; - } else { - // Destroy the TimeStamp object - tsCache_->freeTimeStamp(ts); - command.setData(NULL); - } - } -} - -bool VirtualGPU::profilingCollectResults(CommandBatch* cb, const amd::Event* waitingEvent) { - bool found = false; - amd::Command* current; - amd::Command* first = cb->head_; - - // If the command list is, empty then exit - if (NULL == first) { - return found; - } - - // Wait for the last known GPU events on all engines - waitEventLock(cb); - - // Find the CPU base time of the entire command batch execution - uint64_t endTimeStamp = amd::Os::timeNanos(); - uint64_t startTimeStamp = endTimeStamp; - - // First step, walk the command list to find the first valid command - //! \note The batch may have empty markers at the beginning. - //! So the start/end of the empty commands is equal to - //! the start of the first valid command in the batch. - first = cb->head_; - while (NULL != first) { - // Get the TimeStamp object associated witht the current command - TimeStamp* ts = reinterpret_cast(first->data()); - - if (ts != NULL) { - ts->value(&startTimeStamp, &endTimeStamp); - endTimeStamp -= readjustTimeGPU_; - startTimeStamp -= readjustTimeGPU_; - // Assign to endTimeStamp the start of the first valid command - endTimeStamp = startTimeStamp; - break; - } - first = first->getNext(); - } - - // Second step, walk the command list to construct the time line - first = cb->head_; - while (NULL != first) { - // Get the TimeStamp object associated witht the current command - TimeStamp* ts = reinterpret_cast(first->data()); - - current = first->getNext(); - - if (ts != NULL) { - ts->value(&startTimeStamp, &endTimeStamp); - endTimeStamp -= readjustTimeGPU_; - startTimeStamp -= readjustTimeGPU_; - // Destroy the TimeStamp object - tsCache_->freeTimeStamp(ts); - first->setData(NULL); - } else { - // For empty commands start/end is equal to - // the end of the last valid command - startTimeStamp = endTimeStamp; - } - - // Update the command status with the proper timestamps - if (first->status() == CL_SUBMITTED) { - first->setStatus(CL_RUNNING, startTimeStamp); - first->setStatus(CL_COMPLETE, endTimeStamp); - } else if (first->status() == CL_RUNNING) { - first->setStatus(CL_COMPLETE, endTimeStamp); - } else if ((first->status() != CL_COMPLETE) && (current != NULL)) { - LogPrintfError("Unexpected command status - %d!", first->status()); - } - - // Do we wait this event? - if (first == waitingEvent) { - found = true; - } - - first->release(); - first = current; - } - - return found; -} - -bool VirtualGPU::addVmMemory(const Memory* memory) { - uint* cnt = &cal_.memCount_; - (*cnt)++; - // Reallocate array if kernel uses more memory objects - if (numVmMems_ < *cnt) { - gslMemObject* tmp; - tmp = new gslMemObject[*cnt]; - if (tmp == NULL) { - return false; - } - memcpy(tmp, vmMems_, sizeof(gslMemObject) * numVmMems_); - delete[] vmMems_; - vmMems_ = tmp; - numVmMems_ = *cnt; - } - vmMems_[*cnt - 1] = memory->gslResource(); - - return true; -} - -void VirtualGPU::profileEvent(EngineType engine, bool type) const { - if (NULL == currTs_) { - return; - } - if (type) { - currTs_->begin((engine == SdmaEngine) ? true : false); - } else { - currTs_->end((engine == SdmaEngine) ? true : false); - } -} - -bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address params, - bool nativeMem, std::vector* memList) { - const HSAILKernel& hsaKernel = - static_cast(*(kernel.getDeviceKernel(dev()))); - const amd::KernelSignature& signature = kernel.signature(); - const amd::KernelParameters& kernelParams = kernel.parameters(); - - // Mark the tracker with a new kernel, - // so we can avoid checks of the aliased objects - memoryDependency().newKernel(); - - bool deviceSupportFGS = 0 != dev().isFineGrainedSystem(true); - bool supportFineGrainedSystem = deviceSupportFGS; - FGSStatus status = kernelParams.getSvmSystemPointersSupport(); - switch (status) { - case FGS_YES: - if (!deviceSupportFGS) { - return false; - } - supportFineGrainedSystem = true; - break; - case FGS_NO: - supportFineGrainedSystem = false; - break; - case FGS_DEFAULT: - default: - break; - } - - size_t count = kernelParams.getNumberOfSvmPtr(); - size_t execInfoOffset = kernelParams.getExecInfoOffset(); - bool sync = true; - - amd::Memory* memory = NULL; - // get svm non arugment information - void* const* svmPtrArray = reinterpret_cast(params + execInfoOffset); - for (size_t i = 0; i < count; i++) { - memory = amd::MemObjMap::FindMemObj(svmPtrArray[i]); - if (NULL == memory) { - if (!supportFineGrainedSystem) { - return false; - } else if (sync) { - flushCUCaches(); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } - } else { - Memory* gpuMemory = dev().getGpuMemory(memory); - if (NULL != gpuMemory) { - // Synchronize data with other memory instances if necessary - gpuMemory->syncCacheFromHost(*this); - - const static bool IsReadOnly = false; - // Validate SVM passed in the non argument list - memoryDependency().validate(*this, gpuMemory, IsReadOnly); - - // Mark signal write for cache coherency, - // since this object isn't a part of kernel arg setup - if ((memory->getMemFlags() & CL_MEM_READ_ONLY) == 0) { - memory->signalWrite(&dev()); - } - - memList->push_back(gpuMemory); - } else { - return false; - } - } - } - - amd::Memory* const* memories = - reinterpret_cast(params + kernelParams.memoryObjOffset()); - // Check all parameters for the current kernel - for (size_t i = 0; i < signature.numParameters(); ++i) { - const amd::KernelParameterDescriptor& desc = signature.at(i); - const HSAILKernel::Argument* arg = hsaKernel.argument(i); - Memory* gpuMem = nullptr; - amd::Memory* mem = nullptr; - - // Find if current argument is a buffer - if ((desc.type_ == T_POINTER) && (arg->addrQual_ != HSAIL_ADDRESS_LOCAL)) { - uint32_t index = desc.info_.arrayIndex_; - if (nativeMem) { - gpuMem = reinterpret_cast(memories)[index]; - if (nullptr != gpuMem) { - mem = gpuMem->owner(); - } - } else { - mem = memories[index]; - if (mem != nullptr) { - gpuMem = dev().getGpuMemory(mem); - // Synchronize data with other memory instances if necessary - gpuMem->syncCacheFromHost(*this); - } - } - //! This condition is for SVM fine-grain - if ((gpuMem == nullptr) && dev().isFineGrainedSystem(true)) { - flushCUCaches(); - // Clear memory dependency state - const static bool All = true; - memoryDependency().clear(!All); - continue; - } else if (gpuMem != nullptr) { - bool readOnly = false; - // Check image - readOnly = (desc.accessQualifier_ == CL_KERNEL_ARG_ACCESS_READ_ONLY) ? true : false; - // Check buffer - readOnly |= (arg->access_ == HSAIL_ACCESS_TYPE_RO) ? true : false; - // Validate memory for a dependency in the queue - memoryDependency().validate(*this, gpuMem, readOnly); - } - } - } - - for (gpu::Memory* mem : hsaKernel.prog().globalStores()) { - const static bool IsReadOnly = false; - // Validate global store for a dependency in the queue - memoryDependency().validate(*this, mem, IsReadOnly); - } - - return true; -} - -amd::Memory* VirtualGPU::createBufferFromImage(amd::Memory& amdImage) { - amd::Memory* mem = new (amdImage.getContext()) amd::Buffer(amdImage, 0, 0, amdImage.getSize()); - mem->setVirtualDevice(this); - if ((mem != NULL) && !mem->create()) { - mem->release(); - } - - return mem; -} - -void VirtualGPU::writeVQueueHeader(VirtualGPU& hostQ, uint64_t kernelTable) { - const static bool Wait = true; - vqHeader_->kernel_table = kernelTable; - virtualQueue_->writeRawData(hostQ, sizeof(AmdVQueueHeader), vqHeader_, !Wait); -} - -void VirtualGPU::flushCuCaches(HwDbgGpuCacheMask cache_mask) { - //! @todo: fix issue of no event available for the flush/invalidate cache command - InvalidateSqCaches(cache_mask.sqICache_, cache_mask.sqKCache_, cache_mask.tcL1_, - cache_mask.tcL2_); - - flushDMA(engineID_); - - return; -} - -void VirtualGPU::buildKernelInfo(const HSAILKernel& hsaKernel, hsa_kernel_dispatch_packet_t* aqlPkt, - HwDbgKernelInfo& kernelInfo, amd::Event* enqueueEvent) { - amd::HwDebugManager* dbgManager = dev().hwDebugMgr(); - assert(dbgManager && "No HW Debug Manager!"); - - // Initialize structure with default values - - if (hsaKernel.prog().maxScratchRegs() > 0) { - gpu::Memory* scratchBuf = dev().scratch(hwRing())->memObj_; - kernelInfo.scratchBufAddr = scratchBuf->vmAddress(); - kernelInfo.scratchBufferSizeInBytes = scratchBuf->size(); - - // Get the address of the scratch buffer and its size for CPU access - address scratchRingAddr = static_cast
(scratchBuf->map(NULL, 0)); - dbgManager->setScratchRing(scratchRingAddr, scratchBuf->size()); - scratchBuf->unmap(NULL); - } else { - kernelInfo.scratchBufAddr = 0; - kernelInfo.scratchBufferSizeInBytes = 0; - dbgManager->setScratchRing(NULL, 0); - } - - - //! @todo: need to verify what is wanted for the global memory - kernelInfo.heapBufAddr = (dev().globalMem()).vmAddress(); - - kernelInfo.pAqlDispatchPacket = aqlPkt; - kernelInfo.pAqlQueuePtr = reinterpret_cast(hsaQueueMem_->vmAddress()); - - // Get the address of the kernel code and its size for CPU access - gpu::Memory* aqlCode = hsaKernel.gpuAqlCode(); - if (NULL != aqlCode) { - address aqlCodeAddr = static_cast
(aqlCode->map(NULL, 0)); - dbgManager->setKernelCodeInfo(aqlCodeAddr, hsaKernel.aqlCodeSize()); - aqlCode->unmap(NULL); - } else { - dbgManager->setKernelCodeInfo(NULL, 0); - } - - kernelInfo.trapPresent = false; - kernelInfo.trapHandler = NULL; - kernelInfo.trapHandlerBuffer = NULL; - - kernelInfo.excpEn = 0; - kernelInfo.cacheDisableMask = 0; - kernelInfo.sqDebugMode = 0; - - kernelInfo.mgmtSe0Mask = 0xFFFFFFFF; - kernelInfo.mgmtSe1Mask = 0xFFFFFFFF; - - // set kernel info for HW debug and call the callback function - if (NULL != dbgManager->preDispatchCallBackFunc()) { - DebugToolInfo dbgSetting = {0}; - dbgSetting.scratchAddress_ = kernelInfo.scratchBufAddr; - dbgSetting.scratchSize_ = kernelInfo.scratchBufferSizeInBytes; - dbgSetting.globalAddress_ = kernelInfo.heapBufAddr; - dbgSetting.aclBinary_ = hsaKernel.prog().binaryElf(); - dbgSetting.event_ = enqueueEvent; - - // Call the predispatch callback function & set the trap info - AqlCodeInfo aqlCodeInfo; - aqlCodeInfo.aqlCode_ = (amd_kernel_code_t*)hsaKernel.cpuAqlCode(); - aqlCodeInfo.aqlCodeSize_ = hsaKernel.aqlCodeSize(); - - // Execute the pre-dispatch call back function - dbgManager->executePreDispatchCallBack(reinterpret_cast(aqlPkt), &dbgSetting); - - // assign the debug TMA and TBA for kernel dispatch - if (NULL != dbgSetting.trapHandler_ && NULL != dbgSetting.trapBuffer_) { - assignDebugTrapHandler(dbgSetting, kernelInfo); - } - - kernelInfo.trapPresent = (kernelInfo.trapHandler) ? true : false; - - // Execption policy - kernelInfo.excpEn = dbgSetting.exceptionMask_; - kernelInfo.cacheDisableMask = dbgSetting.cacheDisableMask_; - kernelInfo.sqDebugMode = dbgSetting.gpuSingleStepMode_; - - // Compute the mask for reserved CUs. These two dwords correspond to - // two registers used for reserving CUs for display. In the current - // implementation, the number of CUs reserved can be 0 to 7, and it - // is set by debugger users. - if (dbgSetting.monitorMode_) { - uint32_t i = dbgSetting.reservedCuNum_ / 2; - kernelInfo.mgmtSe0Mask <<= i; - i = dbgSetting.reservedCuNum_ - i; - kernelInfo.mgmtSe1Mask <<= i; - } - - // flush/invalidate the instruction, data, L1 and L2 caches - InvalidateSqCaches(); - } -} - -void VirtualGPU::assignDebugTrapHandler(const DebugToolInfo& dbgSetting, - HwDbgKernelInfo& kernelInfo) { - // setup the runtime trap handler code and trap buffer to be assigned before kernel dispatching - // - Memory* rtTrapHandlerMem = static_cast(dev().hwDebugMgr()->runtimeTBA()); - Memory* rtTrapBufferMem = static_cast(dev().hwDebugMgr()->runtimeTMA()); - - kernelInfo.trapHandler = reinterpret_cast(rtTrapHandlerMem->vmAddress() + TbaStartOffset); - // With the TMA corruption hw bug workaround, the trap handler buffer can be set to zero. - // However, by setting the runtime trap buffer (TMA) correct, the runtime trap hander - // without the workaround can still function correctly. - kernelInfo.trapHandlerBuffer = reinterpret_cast(rtTrapBufferMem->vmAddress()); - - address rtTrapBufferAddress = static_cast
(rtTrapBufferMem->map(this)); - - Memory* trapHandlerMem = dev().getGpuMemory(dbgSetting.trapHandler_); - Memory* trapBufferMem = dev().getGpuMemory(dbgSetting.trapBuffer_); - - // Address of the trap handler code/buffer should be 256-byte aligned - uint64_t tbaAddress = trapHandlerMem->vmAddress(); - uint64_t tmaAddress = trapBufferMem->vmAddress(); - if ((tbaAddress & 0xFF) != 0 || (tmaAddress & 0xFF) != 0) { - assert(false && "Trap handler/buffer is not 256-byte aligned"); - } - - // The addresses of the debug trap handler code (TBA) and buffer (TMA) are - // stored in the runtime trap handler buffer with offset location of 0x18-19 - // and 0x20-21, respectively. - uint64_t* rtTmaPtr = reinterpret_cast(rtTrapBufferAddress + 0x18); - rtTmaPtr[0] = tbaAddress; - rtTmaPtr[1] = tmaAddress; - - rtTrapBufferMem->unmap(NULL); - - // Add GSL handle to the memory list for VidMM - addVmMemory(trapHandlerMem); - addVmMemory(trapBufferMem); - addVmMemory(rtTrapHandlerMem); - addVmMemory(rtTrapBufferMem); -} - -void VirtualGPU::submitTransferBufferFromFile(amd::TransferBufferFileCommand& cmd) { - size_t copySize = cmd.size()[0]; - size_t fileOffset = cmd.fileOffset(); - Memory* mem = dev().getGpuMemory(&cmd.memory()); - uint idx = 0; - - assert((cmd.type() == CL_COMMAND_READ_SSG_FILE_AMD) || - (cmd.type() == CL_COMMAND_WRITE_SSG_FILE_AMD)); - const bool writeBuffer(cmd.type() == CL_COMMAND_READ_SSG_FILE_AMD); - - if (writeBuffer) { - size_t dstOffset = cmd.origin()[0]; - while (copySize > 0) { - Memory* staging = dev().getGpuMemory(&cmd.staging(idx)); - size_t dstSize = amd::TransferBufferFileCommand::StagingBufferSize; - dstSize = std::min(dstSize, copySize); - void* dstBuffer = staging->cpuMap(*this); - if (!cmd.file()->transferBlock(writeBuffer, dstBuffer, staging->size(), fileOffset, 0, - dstSize)) { - cmd.setStatus(CL_INVALID_OPERATION); - return; - } - staging->cpuUnmap(*this); - - bool result = blitMgr().copyBuffer(*staging, *mem, 0, dstOffset, dstSize, false); - flushDMA(getGpuEvent(staging->gslResource())->engineId_); - fileOffset += dstSize; - dstOffset += dstSize; - copySize -= dstSize; - } - } else { - size_t srcOffset = cmd.origin()[0]; - while (copySize > 0) { - Memory* staging = dev().getGpuMemory(&cmd.staging(idx)); - size_t srcSize = amd::TransferBufferFileCommand::StagingBufferSize; - srcSize = std::min(srcSize, copySize); - bool result = blitMgr().copyBuffer(*mem, *staging, srcOffset, 0, srcSize, false); - - void* srcBuffer = staging->cpuMap(*this); - if (!cmd.file()->transferBlock(writeBuffer, srcBuffer, staging->size(), fileOffset, 0, - srcSize)) { - cmd.setStatus(CL_INVALID_OPERATION); - return; - } - staging->cpuUnmap(*this); - - fileOffset += srcSize; - srcOffset += srcSize; - copySize -= srcSize; - } - } -} - -} // namespace gpu diff --git a/projects/clr/rocclr/device/gpu/gpuvirtual.hpp b/projects/clr/rocclr/device/gpu/gpuvirtual.hpp deleted file mode 100644 index a8791f4be2..0000000000 --- a/projects/clr/rocclr/device/gpu/gpuvirtual.hpp +++ /dev/null @@ -1,528 +0,0 @@ -/* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef GPUVIRTUAL_HPP_ -#define GPUVIRTUAL_HPP_ - -#include "device/gpu/gpudefs.hpp" -#include "device/gpu/gpuconstbuf.hpp" -#include "device/gpu/gpuprintf.hpp" -#include "device/gpu/gputimestamp.hpp" -#include "device/gpu/gpusched.hpp" -#include "platform/commandqueue.hpp" -#include "device/blit.hpp" - -#include "device/gpu/gpudebugger.hpp" - - -/*! \addtogroup GPU GPU Resource Implementation - * @{ - */ - -//! GPU Device Implementation -namespace gpu { - -class Device; -class Kernel; -class Memory; -class CalCounterReference; -class VirtualGPU; -class Program; -class BlitManager; -class ThreadTrace; -class HSAILKernel; - -//! Virtual GPU -class VirtualGPU : public device::VirtualDevice, public CALGSLContext { - public: - struct CommandBatch : public amd::HeapObject { - amd::Command* head_; //!< Command batch head - GpuEvent events_[AllEngines]; //!< Last known GPU events - TimeStamp* lastTS_; //!< TS associated with command batch - - //! Constructor - CommandBatch(amd::Command* head, //!< Command batch head - const GpuEvent* events, //!< HW events on all engines - TimeStamp* lastTS //!< Last TS in command batch - ) - : head_(head), lastTS_(lastTS) { - memcpy(&events_, events, AllEngines * sizeof(GpuEvent)); - } - }; - - //! The virtual GPU states - union State { - struct { - uint boundGlobal_ : 1; //!< Global buffer was bound - uint profiling_ : 1; //!< Profiling is enabled - uint forceWait_ : 1; //!< Forces wait in flush() - uint boundCb_ : 1; //!< Constant buffer was bound - uint boundPrintf_ : 1; //!< Printf buffer was bound - uint hsailKernel_ : 1; //!< True if HSAIL kernel was used - }; - uint value_; - State() : value_(0) {} - }; - - //! CAL descriptor for the GPU virtual device - struct CalVirtualDesc : public amd::EmbeddedObject { - gslDomain3D gridBlock; //!< size of a block of data - gslDomain3D gridSize; //!< size of 'blocks' to execute - gslDomain3D partialGridBlock; //!< Partial grid block - CALuint localSize; //!< size of OpenCL Local Memory in bytes - uint memCount_; //!< Memory objects count - GpuEvent events_[AllEngines]; //!< Last known GPU events - uint iterations_; //!< Number of iterations for the execution - TimeStamp* lastTS_; //!< Last timestamp executed on Virtual GPU - gslMemObject constBuffers_[MaxConstBuffers]; //!< Constant buffer names - gslMemObject uavs_[MaxUavArguments]; //!< UAV bindings - gslMemObject readImages_[MaxReadImage]; //!< Read images - uint32_t samplersState_[MaxSamplers]; //!< State of all samplers - }; - - typedef std::vector constbufs_t; - - //! GSL descriptor for the GPU kernel, specific to the virtual device - struct GslKernelDesc : public amd::HeapObject { - CALimage image_; //!< CAL image for the program - gslProgramObject func_; //!< GSL program object - gslMemObject intCb_; //!< Internal constant buffer - }; - - struct ResourceSlot { - union State { - struct { - uint bound_ : 1; //!< Resource is bound - uint constant_ : 1; //!< Resource is a constant - }; - uint value_; - State() : value_(0) {} - }; - - State state_; //!< slot's state - const Memory* memory_; //!< GPU memory object - - ResourceSlot() : memory_(NULL) {} - - //! Copy constructor for the kernel argument - ResourceSlot(const ResourceSlot& data) { *this = data; } - - //! Overloads operator= - ResourceSlot& operator=(const ResourceSlot& data) { - state_.value_ = data.state_.value_; - memory_ = data.memory_; - return *this; - } - }; - - class MemoryDependency : public amd::EmbeddedObject { - public: - //! Default constructor - MemoryDependency() - : memObjectsInQueue_(NULL), endMemObjectsInQueue_(0), numMemObjectsInQueue_(0), maxMemObjectsInQueue_(0) {} - - ~MemoryDependency() { delete[] memObjectsInQueue_; } - - //! Creates memory dependecy structure - bool create(size_t numMemObj); - - //! Notify the tracker about new kernel - void newKernel() { endMemObjectsInQueue_ = numMemObjectsInQueue_; } - - //! Validates memory object on dependency - void validate(VirtualGPU& gpu, const Memory* memory, bool readOnly); - - //! Clear memory dependency - void clear(bool all = true); - - private: - struct MemoryState { - uint64_t start_; //! Busy memory start address - uint64_t end_; //! Busy memory end address - bool readOnly_; //! Current GPU state in the queue - }; - - MemoryState* memObjectsInQueue_; //!< Memory object state in the queue - size_t endMemObjectsInQueue_; //!< End of mem objects in the queue - size_t numMemObjectsInQueue_; //!< Number of mem objects in the queue - size_t maxMemObjectsInQueue_; //!< Maximum number of mem objects in the queue - }; - - - class DmaFlushMgmt : public amd::EmbeddedObject { - public: - DmaFlushMgmt(const Device& dev); - - // Resets DMA command buffer workload - void resetCbWorkload(const Device& dev); - - // Finds split size for the current dispatch - void findSplitSize(const Device& dev, //!< GPU device object - uint64_t threads, //!< Total number of execution threads - uint instructions //!< Number of ALU instructions - ); - - // Returns TRUE if DMA command buffer is ready for a flush - bool isCbReady(VirtualGPU& gpu, //!< Virtual GPU object - uint64_t threads, //!< Total number of execution threads - uint instructions //!< Number of ALU instructions - ); - - // Returns dispatch split size - uint dispatchSplitSize() const { return dispatchSplitSize_; } - - private: - uint64_t maxDispatchWorkload_; //!< Maximum number of operations for a single dispatch - uint64_t maxCbWorkload_; //!< Maximum number of operations for DMA command buffer - uint64_t cbWorkload_; //!< Current number of operations in DMA command buffer - uint aluCnt_; //!< All ALUs on the chip - uint dispatchSplitSize_; //!< Dispath split size in elements - }; - - typedef std::vector ResourceSlots; - - public: - explicit VirtualGPU(Device& device); - bool create(bool profiling, uint rtCUs = amd::CommandQueue::RealTimeDisabled, - uint deviceQueueSize = 0, - amd::CommandQueue::Priority priority = amd::CommandQueue::Priority::Normal); - ~VirtualGPU(); - - void submitReadMemory(amd::ReadMemoryCommand& vcmd); - void submitWriteMemory(amd::WriteMemoryCommand& vcmd); - void submitCopyMemory(amd::CopyMemoryCommand& vcmd); - void submitCopyMemoryP2P(amd::CopyMemoryP2PCommand& vcmd) {} - void submitMapMemory(amd::MapMemoryCommand& vcmd); - void submitUnmapMemory(amd::UnmapMemoryCommand& vcmd); - void submitKernel(amd::NDRangeKernelCommand& vcmd); - bool submitKernelInternal( - const amd::NDRangeContainer& sizes, //!< Workload sizes - const amd::Kernel& kernel, //!< Kernel for execution - const_address parameters, //!< Parameters for the kernel - bool nativeMem = true, //!< Native memory objects - amd::Event* enqueueEvent = NULL //!< Event provided in the enqueue kernel command - ); - bool submitKernelInternalHSA( - const amd::NDRangeContainer& sizes, //!< Workload sizes - const amd::Kernel& kernel, //!< Kernel for execution - const_address parameters, //!< Parameters for the kernel - bool nativeMem = true, //!< Native memory objects - amd::Event* enqueueEvent = NULL //!< Event provided in the enqueue kernel command - ); - void submitNativeFn(amd::NativeFnCommand& vcmd); - void submitFillMemory(amd::FillMemoryCommand& vcmd); - void submitMigrateMemObjects(amd::MigrateMemObjectsCommand& cmd); - void submitMarker(amd::Marker& vcmd); - void submitAcquireExtObjects(amd::AcquireExtObjectsCommand& vcmd); - void submitReleaseExtObjects(amd::ReleaseExtObjectsCommand& vcmd); - void submitPerfCounter(amd::PerfCounterCommand& vcmd); - void submitThreadTraceMemObjects(amd::ThreadTraceMemObjectsCommand& cmd); - void submitThreadTrace(amd::ThreadTraceCommand& vcmd); - void submitSignal(amd::SignalCommand& vcmd); - void submitMakeBuffersResident(amd::MakeBuffersResidentCommand& vcmd); - virtual void submitSvmFreeMemory(amd::SvmFreeMemoryCommand& cmd); - virtual void submitSvmCopyMemory(amd::SvmCopyMemoryCommand& cmd); - virtual void submitSvmFillMemory(amd::SvmFillMemoryCommand& cmd); - virtual void submitSvmMapMemory(amd::SvmMapMemoryCommand& cmd); - virtual void submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& cmd); - virtual void submitTransferBufferFromFile(amd::TransferBufferFileCommand& cmd); - - void releaseMemory(gslMemObject gslResource, bool wait = true); - void releaseKernel(CALimage calImage); - - void flush(amd::Command* list = NULL, bool wait = false); - - //! Returns GPU device object associated with this kernel - const Device& dev() const { return gpuDevice_; } - - //! Returns CAL descriptor of the virtual device - const CalVirtualDesc* cal() const { return &cal_; } - - //! Returns active kernel descriptor for this virtual device - const GslKernelDesc* gslKernelDesc() const { return activeKernelDesc_; } - - //! Returns a GPU event, associated with GPU memory - GpuEvent* getGpuEvent(const gslMemObject gslMem //!< GSL mem object - ) { - return &gpuEvents_[gslMem]; - } - - //! Assigns a GPU event, associated with GPU memory - void assignGpuEvent(const gslMemObject gslMem, //!< GSL mem object - GpuEvent gpuEvent) { - gpuEvents_[gslMem] = gpuEvent; - } - - //! Set the kernel as active - bool setActiveKernelDesc(const amd::NDRangeContainer& sizes, //!< kernel execution work sizes - const Kernel* kernel //!< GPU kernel object - ); - - //! Set the last known GPU event - void setGpuEvent(GpuEvent gpuEvent, //!< GPU event for tracking - bool flush = false //!< TRUE if flush is required - ); - - //! Flush DMA buffer on the specified engine - void flushDMA(uint engineID //!< Engine ID for DMA flush - ); - - //! Wait for all engines on this Virtual GPU - //! Returns TRUE if CPU didn't wait for GPU - bool waitAllEngines(CommandBatch* cb = NULL //!< Command batch - ); - - //! Waits for the latest GPU event with a lock to prevent multiple entries - void waitEventLock(CommandBatch* cb //!< Command batch - ); - - //! Returns a resource associated with the constant buffer - const ConstBuffer* cb(uint idx) const { return constBufs_[idx]; } - - //! Adds CAL objects into the constant buffer vector - void addConstBuffer(ConstBuffer* cb) { constBufs_.push_back(cb); } - - constbufs_t constBufs_; //!< constant buffers - - //! Start the command profiling - void profilingBegin(amd::Command& command, //!< Command queue object - bool drmProfiling = false //!< Measure DRM time - ); - - //! End the command profiling - void profilingEnd(amd::Command& command); - - //! Attach profiler - void profilerAttach(bool enable = false) {} - - //! Collect the profiling results - bool profilingCollectResults(CommandBatch* cb, //!< Command batch - const amd::Event* waitingEvent //!< Waiting event - ); - - //! Adds a memory handle into the GSL memory array for Virtual Heap - bool addVmMemory(const Memory* memory //!< GPU memory object - ); - - //! Adds a stage write buffer into a list - void addXferWrite(Memory& memory); - - //! Adds a pinned memory object into a map - void addPinnedMem(amd::Memory* mem); - - //! Release pinned memory objects - void releasePinnedMem(); - - //! Finds if pinned memory is cached - amd::Memory* findPinnedMem(void* addr, size_t size); - - //! Returns gsl memory object for VM - const gslMemObject* vmMems() const { return vmMems_; } - - //! Get the PrintfDbg object - PrintfDbg& printfDbg() const { return *printfDbg_; } - - //! Get the PrintfDbgHSA object - PrintfDbgHSA& printfDbgHSA() const { return *printfDbgHSA_; } - - //! Enables synchronized transfers - void enableSyncedBlit() const; - - //! Checks if profiling is enabled - bool profiling() const { return state_.profiling_; } - - //! Returns memory dependency class - MemoryDependency& memoryDependency() { return memoryDependency_; } - - //! Returns hsaQueueMem_ - const Memory* hsaQueueMem() const { return hsaQueueMem_; } - - //! Returns DMA flush management structure - const DmaFlushMgmt& dmaFlushMgmt() const { return dmaFlushMgmt_; } - - //! Releases GSL memory objects allocated on this queue - void releaseMemObjects(bool scratch = true); - - //! Returns the HW ring used on this virtual device - uint hwRing() const { return hwRing_; } - - //! Returns current timestamp object for profiling - TimeStamp* currTs() const { return cal_.lastTS_; } - - //! Returns virtual queue object for device enqueuing - Memory* vQueue() const { return virtualQueue_; } - - //! Update virtual queue header - void writeVQueueHeader(VirtualGPU& hostQ, uint64_t kernelTable); - - //! Returns TRUE if virtual queue was successfully allocatted - bool createVirtualQueue(uint deviceQueueSize //!< Device queue size - ); - - EngineType engineID_; //!< Engine ID for this VirtualGPU - ResourceSlots slots_; //!< Resource slots for kernel arguments - State state_; //!< virtual GPU current state - CalVirtualDesc cal_; //!< CAL virtual device descriptor - - void flushCuCaches(HwDbgGpuCacheMask cache_mask); //!< flush/invalidate SQ cache - - protected: - virtual void profileEvent(EngineType engine, bool type) const; - - //! Creates buffer object from image - amd::Memory* createBufferFromImage( - amd::Memory& amdImage //! The parent image object(untiled images only) - ); - - private: - typedef std::unordered_map GslKernels; - typedef std::unordered_map GpuEvents; - - //! Finds total amount of necessary iterations - inline void findIterations(const amd::NDRangeContainer& sizes, //!< Original workload sizes - const amd::NDRange& local, //!< Local workgroup size - amd::NDRange& groups, //!< Calculated workgroup sizes - amd::NDRange& remainder, //!< Calculated remainder sizes - size_t& extra //!< Amount of extra executions for remainder - ); - - //! Setups workloads for the current iteration - inline void setupIteration( - uint iteration, //!< Current iteration - const amd::NDRangeContainer& sizes, //!< Original workload sizes - Kernel& gpuKernel, //!< GPU kernel - amd::NDRange& global, //!< Global size for the current iteration - amd::NDRange& offsets, //!< Offsets for the current iteration - amd::NDRange& local, //!< Local sizes for the current iteration - amd::NDRange& groups, //!< Group sizes for the current iteration - amd::NDRange& groupOffset, //!< Group offsets for the current iteration - amd::NDRange& divider, //!< Group divider - amd::NDRange& remainder, //!< Remain workload - size_t extra //!< Extra groups - ); - - //! Allocates constant buffers - bool allocConstantBuffers(); - - //! Allocates CAL kernel descriptor of the virtual device - GslKernelDesc* allocKernelDesc(const Kernel* kernel, //!< Kernel object - CALimage calImage); //!< CAL image - - //! Frees CAL kernel descriptor of the virtual device - void freeKernelDesc(GslKernelDesc* desc); - - bool gslOpen(uint nEngines, gslEngineDescriptor* engines, uint32_t rtCUs); - void gslDestroy(); - - //! Releases stage write buffers - void releaseXferWrite(); - - //! Allocate hsaQueueMem_ - bool allocHsaQueueMem(); - - //! Awaits a command batch with a waiting event - bool awaitCompletion(CommandBatch* cb, //!< Command batch for to wait - const amd::Event* waitingEvent = NULL //!< A waiting event - ); - - //! Validates the scratch buffer memory for a specified kernel - void validateScratchBuffer(const Kernel* kernel //!< Kernel for validaiton - ); - - //! Detects memory dependency for HSAIL kernels and flushes caches - bool processMemObjectsHSA(const amd::Kernel& kernel, //!< AMD kernel object for execution - const_address params, //!< Pointer to the param's store - bool nativeMem, //!< Native memory objects - std::vector* memList //!< Memory list for KMD tracking - ); - - //! Common function for fill memory used by both svm Fill and non-svm fill - bool fillMemory(cl_command_type type, //!< the command type - amd::Memory* amdMemory, //!< memory object to fill - const void* pattern, //!< pattern to fill the memory - size_t patternSize, //!< pattern size - const amd::Coord3D& origin, //!< memory origin - const amd::Coord3D& size //!< memory size for filling - bool forceBlit = false //!< force shader blit path - ); - - bool copyMemory(cl_command_type type, //!< the command type - amd::Memory& srcMem, //!< source memory object - amd::Memory& dstMem, //!< destination memory object - bool entire, //!< flag of entire memory copy - const amd::Coord3D& srcOrigin, //!< source memory origin - const amd::Coord3D& dstOrigin, //!< destination memory object - const amd::Coord3D& size, //!< copy size - const amd::BufferRect& srcRect, //!< region of source for copy - const amd::BufferRect& dstRect //!< region of destination for copy - ); - - void buildKernelInfo(const HSAILKernel& hsaKernel, //!< hsa kernel - hsa_kernel_dispatch_packet_t* aqlPkt, //!< aql packet for dispatch - HwDbgKernelInfo& kernelInfo, //!< kernel info for the dispatch - amd::Event* enqueueEvent //!< Event provided in the enqueue kernel command - ); - - void assignDebugTrapHandler(const DebugToolInfo& dbgSetting, //!< debug settings - HwDbgKernelInfo& kernelInfo //!< kernel info for the dispatch - ); - - GslKernels gslKernels_; //!< GSL kernel descriptors - GslKernelDesc* activeKernelDesc_; //!< active GSL kernel descriptors - GpuEvents gpuEvents_; //!< GPU events - - Device& gpuDevice_; //!< physical GPU device - - PrintfDbg* printfDbg_; //!< GPU printf implemenation - PrintfDbgHSA* printfDbgHSA_; //!< HSAIL printf implemenation - - TimeStampCache* tsCache_; //!< TimeStamp cache - MemoryDependency memoryDependency_; //!< Memory dependency class - - gslMemObject* vmMems_; //!< Array of GSL memories for VM mode - uint numVmMems_; //!< Number of entries in VM mem array - - DmaFlushMgmt dmaFlushMgmt_; //!< DMA flush management - - std::list xferWriteBuffers_; //!< Stage write buffers - std::list pinnedMems_; //!< Pinned memory list - - typedef std::list CommandBatchList; - CommandBatchList cbList_; //!< List of command batches - - uint hwRing_; //!< HW ring used on this virtual device - - uint64_t readjustTimeGPU_; //!< Readjust time between GPU and CPU timestamps - TimeStamp* currTs_; //!< current timestamp for command - - AmdVQueueHeader* vqHeader_; //!< Sysmem copy for virtual queue header - Memory* virtualQueue_; //!< Virtual device queue - Memory* schedParams_; //!< The scheduler parameters - uint schedParamIdx_; //!< Index in the scheduler parameters buffer - uint deviceQueueSize_; //!< Device queue size - uint maskGroups_; //!< The number of mask groups processed in the scheduler by one thread - - Memory* hsaQueueMem_; //!< Memory for the amd_queue_t object - bool profileEnabled_; //!< Profiling is enabled -}; - -/*@}*/} // namespace gpu - -#endif /*GPUVIRTUAL_HPP_*/ diff --git a/projects/clr/rocclr/device/gpu/gslbe/localdefs b/projects/clr/rocclr/device/gpu/gslbe/localdefs deleted file mode 100644 index 8a7663898a..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/localdefs +++ /dev/null @@ -1,2 +0,0 @@ -OPENCL_DEPTH = $(CAL_DEPTH)/../../../.. -include $(OPENCL_DEPTH)/runtime/runtimedefs diff --git a/projects/clr/rocclr/device/gpu/gslbe/localrules b/projects/clr/rocclr/device/gpu/gslbe/localrules deleted file mode 100644 index 0e4c438ddf..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/localrules +++ /dev/null @@ -1 +0,0 @@ -include $(OPENCL_DEPTH)/runtime/runtimerules diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/promotions/Readme.txt b/projects/clr/rocclr/device/gpu/gslbe/src/promotions/Readme.txt deleted file mode 100644 index 862af5db46..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/promotions/Readme.txt +++ /dev/null @@ -1,2 +0,0 @@ -Promotions directory contains functionality from other staging branches copied -(promoted) into the CAL tree. diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.cpp deleted file mode 100644 index 1416ba3c7e..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.cpp +++ /dev/null @@ -1,222 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - - -#include "EventQueue.h" -#include "query/QueryObject.h" -#include "gsl_ctx.h" - -EventQueue::EventQueue() -{ - m_cs = NULL; - m_queueSize = c_staticQueueSize; - - memset(m_queries,0,sizeof(m_queries)); - memset(m_flushed,0,sizeof(m_flushed)); - - m_latestRetired = 0; - m_headId = m_queueSize - 1 ; - m_tail = 0; -} - -EventQueue::~EventQueue() -{ - for (unsigned int i = 0; i < c_staticQueueSize; i++) - { - assert(m_queries[i] == 0); - } -} - -bool -EventQueue::open(gsCtx* cs, gslQueryTarget target, EQManagerConfig config, uint32 engineMask) -{ - assert((config == EQManager_HIGH) || (config == EQManager_LOW)); - setSlotCount((int) config); - assert((GpuEvent::InvalidID+1) % m_queueSize == 0); - - m_cs = cs; - - m_headId = m_queueSize - 1 ; - m_tail = 0; - m_latestRetired = 0; - m_target = target; - m_engineMask = engineMask; - - for (unsigned int i = 0; i < m_queueSize; i++) - { - m_queries[i] = cs->createQuery(target); - } - - return true; -} - -void -EventQueue::close() -{ - if (!m_cs) // the queue is unintialized. - { - return; - } - - for (unsigned int i = 0; i < m_queueSize; i++) - { - m_cs->destroyQuery(m_queries[i]); - } - memset(m_queries, 0, sizeof(m_queries)); - memset(m_flushed, 0, sizeof(m_flushed)); - m_latestRetired = 0; - m_headId = m_queueSize - 1 ; - m_tail = 0; - m_cs = NULL; -} - -void -EventQueue::begin() -{ - const CALuint slot = m_headId % m_queueSize; - gslErrorCode ec = m_queries[slot]->BeginQuery(m_cs, m_target, 0, m_engineMask); - assert(ec == GSL_NO_ERROR); - - m_flushed[slot] = false; // we've started a query, but it hasn't been checked yet... -} - -uint32 -EventQueue::end() -{ - uint32 ret = m_headId; - const uint32 slot = m_headId % m_queueSize; - - m_queries[slot]->EndQuery(m_cs, 0); - - m_headId++; - m_tail++; - - if (GpuEvent::InvalidID == m_headId) - { - // Flush on an event ID wrap around or when the Queue is going to wrap in - flush(); - //roll numbers back to the beginning - m_latestRetired = 0; - m_headId = m_headId % m_queueSize; - m_tail = m_tail % m_queueSize; - } - return ret; -} - -bool -EventQueue::isDone(uint32 event) -{ - assert((event < GpuEvent::InvalidID) && "illegal event handle"); - // if the event is older the the last known retired event we - // do not need to process it. - if (event <= m_latestRetired) - { - return true; - } - - // if the event is older than the oldest event handle we have - // we synchronize with the oldest event. - if (event < m_tail) - { - return waitForEvent(m_tail, CAL_WAIT_LOW_CPU_UTILIZATION); - } - - // - // If we've never called flush on the query object, go ahead flush the first time to ensure - // we never infinite loop - // - const uint32 slot = event % m_queueSize; - if (!m_flushed[slot]) - { - flush(); - } - - // - // Since we're in between, we actually have to check to see if things are truely done - // - bool retVal = m_queries[slot]->IsResultAvailable(m_cs); - - // cache the most recently retired event - if (retVal && (event < m_headId) && (event > m_latestRetired)) - { - m_latestRetired = event; - } - - return retVal; -} - -bool -EventQueue::waitForEvent(uint32 event, uint32 waitType) -{ - // if we already retired a younger event we don't to process current events - if (event <= m_latestRetired) - { - return true; - } - - // if the event is older than the oldest event handle we have - // we synchronize with the oldest event - if (event < m_tail) - { - event = m_tail; - } - - // - // If we've never called flush on the query object, go ahead flush the first time to ensure - // we never infinite loop - // - const uint32 slot = event % m_queueSize; - if (!m_flushed[slot]) - { - flush(); - } - uint64 param; - m_queries[slot]->GetResult(m_cs, ¶m, (IOSyncWaitType) waitType); - - // cache the most recently retired event - if ((event < m_headId) && (event > m_latestRetired)) - { - m_latestRetired = event; - } - - return (param != 0); -} - -bool -EventQueue::flush() -{ - m_cs->Flush(false, m_engineMask); - memset(m_flushed, 1, sizeof(m_flushed)); - return true; -} - - -void -EventQueue::setSlotCount(uint32 slotCount) -{ - if (slotCount < c_staticQueueSize) - { - m_queueSize = slotCount; - } - else - { - m_queueSize = c_staticQueueSize; - } -} diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.h deleted file mode 100644 index 4d90af96a4..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/EventQueue.h +++ /dev/null @@ -1,74 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __EventQueue_h__ -#define __EventQueue_h__ - -#include "backend.h" -#include "atitypes.h" -#include "gsl_types.h" -#include "gsl_config.h" - -namespace gsl -{ - class gsCtx; -}; - -enum EQManagerConfig -{ - EQManager_HIGH = 512, - EQManager_LOW = 32 -}; - -class EventQueue { -public: - static constexpr unsigned int c_staticQueueSize = EQManager_HIGH; - EventQueue(); - ~EventQueue(); - - bool open(gsl::gsCtx* cs, gslQueryTarget target, EQManagerConfig config, uint32 engineMask = GSL_ENGINEMASK_ALL_BUT_UVD_VCE); - void close(); - - void begin(); - uint32 end(); - bool isDone(uint32 event); - bool waitForEvent(uint32 event, uint32 waitType); - bool flush(); - -private: - - gsl::gsCtx* m_cs; - - uint32 m_queueSize; - gslQueryTarget m_target; - uint32 m_engineMask; // EngineMask for this Query - uint32 m_tail; //represents the oldest event we have - uint32 m_headId; - uint32 m_latestRetired; //!< most recentyl retired event. - gslQueryObject m_queries[c_staticQueueSize]; - bool m_flushed[c_staticQueueSize]; - /////////////////////// - // private functions // - /////////////////////// - void setSlotCount(uint32 slotCount); -}; - -#endif - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.cpp deleted file mode 100644 index 1be870fb20..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.cpp +++ /dev/null @@ -1,601 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "gsl_adaptor.h" -#include "GSLContext.h" -#include "GSLDevice.h" -#include "cm_if.h" -#include "amuABI.h" -#include "shader/ProgramObject.h" -#include "shader/ComputeProgramObject.h" -#include "query/QueryObject.h" -#include "query/PerformanceQueryObject.h" -#include "constbuffer/ConstantBufferObject.h" -#include "sampler/SamplerObject.h" -#include "texture/TextureResourceObject.h" -#include "uav/UAVObject.h" -#include "RenderStateObject.h" -#include "shadertracebuffer/ShaderTraceBufferObject.h" -#include "scratchbuffer/ScratchBufferObject.h" -#include "memory/MemObject.h" - -#include - -CALGSLContext::CALGSLContext() -{ - m_cs = 0; - m_rs = 0; - m_allowDMA = false; - - COMPILE_TIME_ASSERT((int)MAX_OUTPUTS <= (int)GSL_MAX_OUTPUT); - - memset(m_textureSamplers, 0, sizeof(m_textureSamplers)); - memset(m_textureResources, 0, sizeof(m_textureResources)); - memset(m_uavResources, 0, sizeof(m_uavResources)); - memset(m_constantBuffers, 0, sizeof(m_constantBuffers)); - - m_scratchBuffers = 0; - - m_waitType = CAL_WAIT_LOW_CPU_UTILIZATION; -} - -CALGSLContext::~CALGSLContext() -{ - assert(m_cs == 0); -} - -bool -CALGSLContext::open( - const CALGSLDevice* pDeviceObject, - uint32 nEngines, - gslEngineDescriptor* engines, - uint32 rtCUs) -{ - m_Dev = pDeviceObject; - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(dev()->gslDeviceOps()); - - gsl::gsAdaptor* native = dev()->getNative(); - assert(native != 0); - - EQManagerConfig EQConfig = EQManager_HIGH; - - gslEngineID mainEngineOrdinal = GSL_ENGINEID_INVALID; - gslEngineID sdmaOrdinal = GSL_ENGINEID_INVALID; - for (uint i = 0; i < nEngines; i++) - { - if (engines[i].id >= GSL_ENGINEID_3DCOMPUTE0 && - engines[i].id <= GSL_ENGINEID_COMPUTE_MEDIUM_PRIORITY) - { - mainEngineOrdinal = engines[i].id; - } - - if (engines[i].id == GSL_ENGINEID_DRMDMA0|| - engines[i].id == GSL_ENGINEID_DRMDMA1) - { - sdmaOrdinal = engines[i].id; - m_allowDMA = dev()->canDMA(); - } - } - - m_cs = native->createComputeContext(mainEngineOrdinal, sdmaOrdinal, false, rtCUs); - - if (m_cs == 0) - { - return false; - } - m_cs->getMainSubCtx()->setVPUMask(dev()->getVPUMask()); - - m_cs->makeCurrent(0); - - m_rs = m_cs->createRenderState(); - if (m_rs == 0) - { - native->deleteContext(m_cs); - m_cs = 0; - return false; - } - - m_cs->setRenderState(m_rs); - m_cs->createSubAllocDesc(); - - // - // - // configure the default compute mode - // - m_rs->setComputeShader(m_cs, true); - - m_eventQueue[MainEngine].open(m_cs, GSL_SYNC_ATI, EQConfig); - m_eventQueue[SdmaEngine].open(m_cs, GSL_DRMDMA_SYNC_ATI, EQConfig, GSL_ENGINE_MASK(GSL_ENGINEID_DRMDMA0) | GSL_ENGINE_MASK(GSL_ENGINEID_DRMDMA1)); - - m_cs->setGPU((gslGPUMask)dev()->getVPUMask()); - - m_cs->setDMAFlushBuf(dev()->m_srcDRMDMAMem, dev()->m_dstDRMDMAMem, 4 /* size of CM_SURF_FMT_R32F*/); - - // Create the GSL scratch buffer object - m_scratchBuffers = m_cs->createScratchBuffer(); - if (m_scratchBuffers == NULL) - { - return false; - } - - if (m_textureSamplers[0] == 0) - { - // Special case. GSL validation requires a sampler with any texture setup. - // In OCL kernel may have an image argument, but doesn't use it. So a sampler - // can be undefined. - //! @note HSAIL will need a sampler as well - m_textureSamplers[0] = m_cs->createSampler(); - m_rs->setSamplerObject(GSL_COMPUTE_PROGRAM, m_textureSamplers[0], 0); - } - - return true; -} - -void -CALGSLContext::close(gsl::gsAdaptor* native) -{ - if (m_cs == 0) - { - return; - } - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(dev()->gslDeviceOps()); - - m_cs->Flush(); - - assert(m_rs != 0); - - m_cs->setRenderState(m_rs); - - m_rs->setCurrentProgramObject(GSL_COMPUTE_PROGRAM, 0); - - for (int i = 0; i < MAX_SAMPLERS; i++) - { - m_rs->setSamplerObject(GSL_COMPUTE_PROGRAM, 0, i); - if (m_textureSamplers[i] != 0) - { - m_cs->destroySampler(m_textureSamplers[i]); - } - } - - for (int i = 0; i < MAX_RESOURCES; i++) - { - m_rs->setTextureResourceObject(m_cs, GSL_COMPUTE_PROGRAM, 0, i); - if (m_textureResources[i] != 0) - { - m_cs->destroyTextureResource(m_textureResources[i]); - } - } - - for (int i = 0; i < MAX_UAVS; i++) - { - m_rs->setUavObject(m_cs, GSL_COMPUTE_PROGRAM, 0, static_cast(GSL_UAV0 + i)); - if (m_uavResources[i] != 0) - { - m_cs->destroyUAVObject(m_uavResources[i]); - } - } - - for (int i = 0; i < MAX_CONSTANTBUFFERS; i++) - { - m_rs->setConstantBufferObject(GSL_COMPUTE_PROGRAM, 0, i); - if (m_constantBuffers[i]) - { - m_cs->destroyConstantBuffer(m_constantBuffers[i]); - } - } - - if (m_scratchBuffers != NULL) - { - //!@todo it should be GSL_COMPUTE_PROGRAM - m_rs->setScratchBufferObject(GSL_FRAGMENT_PROGRAM, 0); - m_scratchBuffers->setMemObject(m_cs, 0, 0); - m_cs->destroyScratchBuffer(m_scratchBuffers); - m_scratchBuffers = 0; - } - - m_cs->setRenderState(0); - - m_cs->destroyRenderState(m_rs); - m_cs->destroySubAllocDesc(); - - m_rs = 0; - - for (uint32 i = 0; i < AllEngines; ++i) - { - m_eventQueue[i].close(); - } - - native->deleteContext(m_cs); - - m_cs = 0; -} - -bool -CALGSLContext::setInput(uint32 physUnit, gslMemObject mem) -{ - assert(physUnit < MAX_RESOURCES); - - //if there is no texture resource object associated with this unit, then allocate one. - if (m_textureResources[physUnit] == 0) - { - m_textureResources[physUnit] = m_cs->createTextureResource(); - m_rs->setTextureResourceObject(m_cs, GSL_COMPUTE_PROGRAM, - m_textureResources[physUnit], physUnit); - } - - m_textureResources[physUnit]->updateDepthTextureParam(mem); - m_textureResources[physUnit]->setMemObject(m_cs, GSL_COMPUTE_PROGRAM, mem, NULL); - - if (mem != NULL) - { - intp channelOrder = mem->getAttribs().channelOrder; - dev()->convertInputChannelOrder(&channelOrder); - m_rs->setTextureResourceSwizzle(GSL_COMPUTE_PROGRAM, physUnit, - reinterpret_cast(&channelOrder)); - } - - return true; -} - -bool -CALGSLContext::setConstantBuffer(uint32 physUnit, gslMemObject mem, uint32 offset, size_t size) -{ - assert(physUnit < MAX_CONSTANTBUFFERS); - assert((physUnit < MAX_APICONSTANTBUFFERS) || (physUnit == SC_INFO_CONSTANTBUFFER)); - - //if there is no constant buffer object associated with this unit, then allocate one. - if (m_constantBuffers[physUnit] == 0) - { - m_constantBuffers[physUnit] = m_cs->createConstantBuffer(); - m_rs->setConstantBufferObject(GSL_COMPUTE_PROGRAM, m_constantBuffers[physUnit], physUnit); - } - - return m_constantBuffers[physUnit]->bindMemory(m_cs, mem, static_cast(offset), (uint32)size); -} - -bool -CALGSLContext::setUAVBuffer(uint32 physUnit, gslMemObject mem, gslUAVType uavType) -{ - assert(physUnit < MAX_UAVS); - - if (m_uavResources[physUnit] == 0) - { - m_uavResources[physUnit] = m_cs->createUAVObject(); - m_rs->setUavObject(m_cs, GSL_COMPUTE_PROGRAM, m_uavResources[physUnit], GSL_UAV0 + physUnit); - } - m_uavResources[physUnit]->setMemObject(m_cs, mem, uavType); - m_uavResources[physUnit]->setRSOBindings(m_cs, GSL_COMPUTE_PROGRAM); - - return true; -} - -void -CALGSLContext::setUAVChannelOrder(uint32 physUnit, gslMemObject mem) -{ - assert(physUnit < MAX_UAVS); - intp channelOrder = mem->getAttribs().channelOrder; - dev()->convertInputChannelOrder(&channelOrder); - m_uavResources[physUnit]->setParameter(GSL_UAV_RESOURCE_SWIZZLE, &channelOrder); -} - -bool -CALGSLContext::isDone(GpuEvent* event) -{ - if (event->isValid()) - { - assert(event->engineId_ < AllEngines); - if (m_eventQueue[event->engineId_].isDone(event->id)) - { - event->invalidate(); - return true; - } - return false; - } - return true; -} - -void -CALGSLContext::waitForEvent(GpuEvent* event) -{ - if (event->isValid()) - { - assert(event->engineId_ < AllEngines); - m_eventQueue[event->engineId_].waitForEvent(event->id, m_waitType); - event->invalidate(); - } -} - -void -CALGSLContext::flushCUCaches(bool flushL2) const -{ - m_cs->FlushCUCaches(flushL2); -} - -void -CALGSLContext::setScratchBuffer(gslMemObject mem, int32 engineId) -{ - // This card has global scratch buffer, so we only manage one resource, - // independent of program type and number of shader engineers. - // For consistency with GSL, We will store the buffer under the - // fragment program type for shader engine 0. - //!@tod should be GSL_COMPUTE_PROGRAM - gslProgramTargetEnum target = GSL_FRAGMENT_PROGRAM; - gslScratchBufferObject scratchBuff = (mem != NULL) ? m_scratchBuffers : NULL; - - m_rs->setScratchBufferObject(target, m_scratchBuffers); - - m_scratchBuffers->setMemObject(m_cs, mem, engineId); -} - -bool -CALGSLContext::copyPartial(GpuEvent& event, - gslMemObject srcMem, - size_t* srcOffset, - gslMemObject destMem, - size_t* destOffset, - size_t* size, - CALmemcopyflags flags, - bool enableRectCopy, - uint32 bytesPerElement) -{ - uint32 mode = GSL_SYNCUPLOAD_IGNORE_ELEMENTSIZE; - EngineType engineId = MainEngine; - assert(m_cs != 0); - uint64 linearBytePitch = 0; - intp bpp = 0; - - CopyType type = dev()->GetCopyType(srcMem, destMem, srcOffset, destOffset, m_allowDMA, - flags, size[0], enableRectCopy); - - if(type == USE_NONE) - { - return false; - } - - switch (flags) - { - case CAL_MEMCOPY_DEFAULT: - case CAL_MEMCOPY_SYNC: - mode |= GSL_SYNCUPLOAD_SYNC_START | GSL_SYNCUPLOAD_SYNC_WAIT; - break; - - case CAL_MEMCOPY_ASYNC: - if ((type == USE_DRMDMA) || (type == USE_DRMDMA_T2L) || (type == USE_DRMDMA_L2T)) - { - engineId = SdmaEngine; - } - break; - - default: - break; - } - - gslErrorCode gslErr = GSL_NO_ERROR; - - switch (type) - { - case USE_DRMDMA: - mode |= GSL_SYNCUPLOAD_DMA; - eventBegin(engineId); - if(enableRectCopy) - { - if ((*srcOffset%4 != 0) || (*destOffset%4 != 0) || (size[0]%4 !=0)) - { - return false; - } - m_cs->syncUploadRawRect(srcMem, srcOffset[0], (uint32)srcOffset[1], (uint32)srcOffset[2], - destMem, destOffset[0], (uint32)destOffset[1], (uint32)destOffset[2], - size[0], (uint32)size[1], (uint32)size[2], mode, bytesPerElement); - } - else - { - m_cs->syncUploadRaw(srcMem, srcOffset[0], destMem, destOffset[0], size[0], mode); - } - eventEnd(engineId, event); - break; - - case USE_DRMDMA_T2L: - mode |= GSL_SYNCUPLOAD_DMA; - eventBegin(engineId); - bpp = srcMem->getBitsPerElement(); - linearBytePitch = size[0] * (bpp / 8); - gslErr = m_cs->DMACopySubSurface(srcOffset[0], (uint32)srcOffset[1], size[0], (uint32)size[1], - destMem, destOffset[0], linearBytePitch, srcMem, 0, 0, ATIGL_FALSE, mode); - eventEnd(engineId, event); - break; - - case USE_DRMDMA_L2T: - mode |= GSL_SYNCUPLOAD_DMA; - eventBegin(engineId); - bpp = destMem->getBitsPerElement(); - linearBytePitch = size[0] * (bpp / 8); - gslErr = m_cs->DMACopySubSurface(destOffset[0], (uint32)destOffset[1], size[0], (uint32)size[1], - srcMem, srcOffset[0], linearBytePitch, destMem, 0, 0, ATIGL_TRUE, mode); - eventEnd(engineId, event); - break; - - case USE_CPDMA: - eventBegin(MainEngine); - m_cs->syncUploadRaw(srcMem, srcOffset[0], destMem, destOffset[0], size[0], mode); - eventEnd(MainEngine, event); - break; - - default: - assert(0); - // - // XXX - should never be here - // - return false; - } - - if (gslErr != GSL_NO_ERROR) - { - return false; - } - - return true; -} - -void -CALGSLContext::setSamplerParameter(uint32 sampler, gslTexParameterPname param, void* vals) -{ - if (m_textureSamplers[sampler] == 0) - { - m_textureSamplers[sampler] = m_cs->createSampler(); - m_rs->setSamplerObject(GSL_COMPUTE_PROGRAM, m_textureSamplers[sampler], sampler); - } - - float* params = reinterpret_cast(vals); - switch (param) - { - case GSL_TEXTURE_MIN_FILTER: - m_textureSamplers[sampler]->setMinFilter(m_cs, - static_cast((uint32)params[0])); - break; - case GSL_TEXTURE_MAG_FILTER: - m_textureSamplers[sampler]->setMagFilter(m_cs, - static_cast((uint32)params[0])); - break; - case GSL_TEXTURE_WRAP_S: - case GSL_TEXTURE_WRAP_T: - case GSL_TEXTURE_WRAP_R: - m_textureSamplers[sampler]->setWrap(m_cs, param, - static_cast((uint32)params[0])); - break; - case GSL_TEXTURE_BORDER_COLOR: - m_textureSamplers[sampler]->setBorderColor(m_cs, params); - break; - default: - assert(!"Unknown sampler state"); - break; - } -} - -bool -CALGSLContext::moduleLoad(CALimage image, - gslProgramObject* func, gslMemObject* constants) -{ - AMUabiMultiBinary binary; - AMUabiEncoding encoding; - - amuABIMultiBinaryCreate(&binary); - amuABIMultiBinaryUnpack(binary, image); - - CALuint machine, type, count = 0; - amuABIMultiBinaryGetEncodingCount(&count, binary); - bool binaryFound = false; - for (CALuint i = 0; i < count; ++i) - { - if (amuABIMultiBinaryGetEncoding(&encoding, binary, i) && - amuABIEncodingGetSignature(&machine, &type, encoding) && - (machine == dev()->getElfMachine()) && (type == (CALuint)ED_ATI_CAL_TYPE_COMPUTE)) - { - binaryFound = true; - break; - } - } - - if (!binaryFound) - { - amuABIMultiBinaryDestroy(binary); - return false; - } - - *func = m_cs->createProgramObject(GSL_COMPUTE_PROGRAM); - if (*func == 0) - { - amuABIMultiBinaryDestroy(binary); - return false; - } - (*func)->programStringARB(m_cs, GSL_COMPUTE_PROGRAM, GSL_PROGRAM_FORMAT_ELF_BINARY, 0, image); - - // Setup the loop constants from the ELF binary int const area. - CALuint numConstants = 0; - CALuint maxPhysical = 0; - - AMUabiLiteralConst* litConsts; - CALuint litConstsCount = 0; - amuABIEncodingGetLitConsts(&litConstsCount, &litConsts, encoding); - for (CALuint i = 0; i < litConstsCount; ++i) - { - if (litConsts[i].type == AMU_ABI_INT32) - { - maxPhysical = std::max(maxPhysical, litConsts[i].addr); - ++numConstants; - } - } - - if (numConstants > 0) - { - const gslMemObjectAttribs attribs( - GSL_MOA_CONSTANT_STORE, // type - GSL_MOA_MEMORY_CARD, // location - GSL_MOA_TILING_LINEAR, // tiling - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_NORM, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED // alloc_type - ); - - *constants = m_cs->createMemObject1D(CM_SURF_FMT_RGBX8, ++maxPhysical, &attribs); - - CALuint* ptr = static_cast((*constants)->map(m_cs, GSL_MAP_READ_WRITE)); - assert(ptr != 0 && "gslMapMemImage failed!"); - - for (CALuint i = 0; i < litConstsCount; ++i) - { - if (litConsts[i].type == AMU_ABI_INT32) - { - ptr[litConsts[i].addr] = litConsts[i].value.int32[0]; - } - } - - (*constants)->unmap(m_cs); - } - - amuABIMultiBinaryDestroy(binary); - - // FIXME Until we get everything right, return an error or we'll hang the HW - return true; -} - -void -CALGSLContext::InvalidateSqCaches(bool instInvalidate, bool dataInvalidate, bool tcL1, bool tcL2) -{ - // invalidating instruction/data L1 caches using Escape - if (instInvalidate || dataInvalidate) { - m_cs->invalidateSqCaches(instInvalidate, dataInvalidate); - } - - if (tcL1) { - flushCUCaches(tcL2); - } - -} - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.h deleted file mode 100644 index 1e61051bb2..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLContext.h +++ /dev/null @@ -1,126 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __GSLContext_h__ -#define __GSLContext_h__ - -#include "atitypes.h" -#include "gsl_types.h" -#include "backend.h" - -#include "EventQueue.h" -#include "amuABI.h" - -#define SC_INFO_CONSTANTBUFFER (147-128) -#define SC_SR_INIT_CONSTANTBUFFER 0 - -#define HW_R800_MAX_UAV 12 -#define SC_R800_ARENA_UAV_SHORT_ID 9 -#define SC_R800_ARENA_UAV_BYTE_ID 10 -#define SC_R800_ARENA_UAV_DWORD_ID 11 - -class CALGSLDevice; - -namespace gsl -{ - class gsAdaptor; -}; - -class CALGSLContext -{ -public: - CALGSLContext(); - ~CALGSLContext(); - - bool open(const CALGSLDevice* pDeviceObject, uint32 nEngines, gslEngineDescriptor *engines, uint32 rtCUs = 0); - void close(gsl::gsAdaptor* native); - - bool setInput(uint32 physUnit, gslMemObject mem); - bool setOutput(uint32 physUnit, gslMemObject mem); - bool setConstantBuffer(uint32 physUnit, gslMemObject mem, CALuint offset, size_t size); - bool setUAVBuffer(uint32 physUnit, gslMemObject mem, gslUAVType uavType); - void setUAVChannelOrder(uint32 physUnit, gslMemObject mem); - bool isDone(GpuEvent* event); - void waitForEvent(GpuEvent* event); - void flushCUCaches(bool flushL2 = false) const; - void eventBegin(EngineType engId) - { - m_eventQueue[engId].begin(); - const static bool Begin = true; - profileEvent(engId, Begin); - } - void eventEnd(EngineType engId, GpuEvent& event) - { - const static bool End = false; - profileEvent(engId, End); - event.id = m_eventQueue[engId].end(); - event.engineId_ = engId; - } - - bool copyPartial(GpuEvent& event, gslMemObject srcMem, size_t* srcOffset, - gslMemObject destMem, size_t* destOffset, size_t* size, CALmemcopyflags flags, bool enableRectCopy, uint32 bytesPerElement); - - void setSamplerParameter(uint32 sampler, gslTexParameterPname param, CALvoid* vals); - - bool recompileShader(CALimage srcImage, CALimage* newImage, const CALuint type); - bool getMachineType(CALuint* pMachine, CALuint* pType, CALimage image); - - bool moduleLoad(CALimage image, gslProgramObject* func, gslMemObject* constants); - - gsl::gsCtx* cs() const { return m_cs; } - gslRenderState rs() const { return m_rs; } - - /// HW Debug support functions - void InvalidateSqCaches(bool instInvalidate = true, bool dataInvalidate = true, bool tcL1 = true, bool tcL2 = true); - -protected: - void setScratchBuffer(gslMemObject mem, int32 engineId); - virtual void profileEvent(EngineType engine, bool type) const {} - - CALwaitType m_waitType; //!< Wait type - -private: - enum { - MAX_OUTPUTS = 12, - MAX_CONSTANTBUFFERS = 20, - MAX_APICONSTANTBUFFERS = 16, - MAX_SAMPLERS = 16, - MAX_RESOURCES = 128, - MAX_SCRATCHBUFFERS = 1, - MAX_SHADERENGINES = 4, - MAX_UAVS = 1024, - }; - - const CALGSLDevice* m_Dev; - const CALGSLDevice* dev() const { return m_Dev; } - - gsl::gsCtx* m_cs; - gslRenderState m_rs; - gslConstantBufferObject m_constantBuffers[MAX_CONSTANTBUFFERS]; - gslUAVObject m_uavResources[MAX_UAVS]; - gslTextureResourceObject m_textureResources[MAX_RESOURCES]; - gslSamplerObject m_textureSamplers[MAX_SAMPLERS]; - gslDrawBuffers m_drawBuffers; - gslScratchBufferObject m_scratchBuffers; - EventQueue m_eventQueue[AllEngines]; - bool m_allowDMA; -}; - -#endif // __GSLContext_h__ diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.cpp deleted file mode 100644 index c11797636f..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.cpp +++ /dev/null @@ -1,1662 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "GSLDevice.h" -#include "EventQueue.h" -#include "ini_export.h" -#include "GSLContext.h" -#include "cm_if.h" -#include "utils/flags.hpp" -#include "query/QueryObject.h" -#include "memory/MemObject.h" -#include "sampler/SamplerObject.h" -#include "texture/TextureResourceObject.h" -#include "../iol/iodrv_if.h" - -extern gslMemObjectAttribTiling g_CALBETiling_Tiled; - -void -CALGSLDevice::resFree(gslMemObject mem) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - m_cs->destroyMemObject(mem); -} - -void CALGSLDevice::Initialize() -{ - m_adp = 0; - m_cs = 0; - m_rs = 0; - m_textureResource = 0; - m_textureSampler = 0; - m_target = (CALtarget)0xffffffff; - m_srcDRMDMAMem = NULL ; - m_dstDRMDMAMem = NULL ; - m_flags = 0; - m_nativeDisplayHandle = NULL; - m_deviceMode = GSL_DEVICE_MODE_GFX; - m_gpuIndex = 0; - m_chainIndex = 0; - m_forcedComputeEngineID = GSL_ENGINEID_INVALID; - m_vpuMask = 1; - gslDeviceOps_ = NULL; -} - -CALGSLDevice::CALGSLDevice() -{ - Initialize(); -} - -CALGSLDevice::~CALGSLDevice() -{ - assert(m_adp == 0); /// CALBE client must call close explicitly. Check that here - - if (m_scfg.sclkActivityThresholdPtr.hasValue) { - osMemFree(m_scfg.sclkActivityThresholdPtr.value); - } - if (m_scfg.sclkDownHysteresisPtr.hasValue) { - osMemFree(m_scfg.sclkDownHysteresisPtr.value); - } - if (m_scfg.sclkUpHysteresisPtr.hasValue) { - osMemFree(m_scfg.sclkUpHysteresisPtr.value); - } - if (m_scfg.packagePowerLimitPtr.hasValue) { - osMemFree(m_scfg.packagePowerLimitPtr.value); - } - if (m_scfg.mclkActivityThresholdPtr.hasValue) { - osMemFree(m_scfg.mclkActivityThresholdPtr.value); - } - if (m_scfg.mclkUpHysteresisPtr.hasValue) { - osMemFree(m_scfg.mclkUpHysteresisPtr.value); - } - if (m_scfg.mclkDownHysteresisPtr.hasValue) { - osMemFree(m_scfg.mclkDownHysteresisPtr.value); - } - - delete gslDeviceOps_; -} - - -gsl::gsAdaptor* -CALGSLDevice::getNative() const -{ - return m_adp; -} - -uint32 -CALGSLDevice::getMaxTextureSize() const -{ - return static_cast(m_maxtexturesize); -} - -void -CALGSLDevice::getAttribs_int(gsl::gsCtx* cs) -{ - m_attribs.struct_size = sizeof(CALdeviceattribs); - - m_attribs.target = m_target; - - gslMemInfo memInfo; - cs->getMemInfo(&memInfo, GSL_MEMINFO_BASIC); - - m_attribs.localRAM = (uint32)((memInfo.cardMemTotalBytes + memInfo.cardExtMemTotalBytes) / (1024 * 1024)); - m_attribs.uncachedRemoteRAM = (uint32)(memInfo.agpMemTotalBytes / (1024 * 1024)); - m_attribs.cachedRemoteRAM = (uint32)(memInfo.agpMemTotalCacheableBytes / (1024 * 1024)); - m_attribs.totalVisibleHeap = (uint32) (memInfo.cardMemTotalBytes / (1024 * 1024)); - m_attribs.totalInvisibleHeap = (uint32) (memInfo.cardExtMemTotalBytes / (1024 * 1024)); - m_attribs.totalDirectHeap = (uint32) (memInfo.directTotalBytes / (1024 * 1024)); - m_attribs.totalCoherentHeap = (uint32) (memInfo.coherentTotalBytes / (1024 * 1024)); - m_attribs.totalRemoteSharedHeap = (uint32) (memInfo.sharedTotalBytes / (1024 * 1024)); - m_attribs.totalCachedRemoteSharedHeap = (uint32) (memInfo.sharedCacheableTotalBytes / (1024 * 1024)); - m_attribs.totalSDIHeap = (uint32) (memInfo.busAddressableTotalBytes / (1024 * 1024)); - - m_attribs.engineClock = cs->getMaxEngineClock(); - m_attribs.memoryClock = cs->getMaxMemoryClock(); - m_attribs.numberOfSIMD = cs->getNumSIMD(); - m_attribs.numberOfCUsperShaderArray = cs->getNumCUsPerShaderArray(); - m_attribs.wavefrontSize = cs->getWaveFrontSize(); - m_attribs.doublePrecision = cs->getIsDoublePrecisionSupported(); - m_attribs.memBusWidth = cs->getVramBitWidth(); - m_attribs.numMemBanks = cs->getVramBanks(); - m_attribs.isWorkstation = cs->getIsWorkstation(); - - m_attribs.numberOfShaderEngines = cs->getNumShaderEngines(); - m_attribs.pciTopologyInformation = m_adp->getLocationId(); - - const uint8* boardName = cs->getString(GSL_GS_RENDERER); - ::strncpy(m_attribs.boardName, (char*)boardName, CAL_ASIC_INFO_MAX_LEN * sizeof(char) - 1); - - const uint8* driverStore = cs->getString(GSL_GS_DRIVER_STORE_PATH); - ::strncpy(m_attribs.driverStore, (char*)driverStore, CAL_DRIVER_STORE_MAX_LEN * sizeof(char) - 1); - - m_attribs.counterFreq = cs->getCounterFreq(); - m_attribs.nanoSecondsPerTick = 1000000000.0 / cs->getCounterFreq(); - m_attribs.longIdleDetect = cs->getLongIdleDetect(); - m_attribs.svmAtomics = m_adp->pAsicInfo->svmAtomics; - - m_attribs.vaStart = static_cast(m_adp->pAsicInfo->vaStart); - m_attribs.vaEnd = static_cast(m_adp->pAsicInfo->vaEnd); - m_attribs.numOfVpu = m_adp->pAsicInfo->numberOfVPU; - m_attribs.isOpenCL200Device = m_adp->pAsicInfo->bIsOpen2Device; - m_attribs.isSVMFineGrainSystem = m_adp->pAsicInfo->svmFineGrainSystem; - m_attribs.isWDDM2Enabled = m_adp->pAsicInfo->vaAvailable && m_adp->pAsicInfo->bNoVATranslation; - m_attribs.maxRTCUs = cs->getMaxRTCUs(); - m_attribs.asicRevision = cs->getChipRev(); - m_attribs.pcieDeviceID = cs->getAsicDID(); - m_attribs.pcieRevisionID = cs->getPciRevID(); -} - -// Parses a single unsigned integer from a comma separated string of integers such as -// 0x24, 0x9235, 0x123,... -// -// pStr (in) The input string -// pValue (out) The unsigned integer output when parsing is successful -// -// Return Pointer to next location in string following a comma. If no such location is found -// then pointer is null. -static char* getNextValue(char* pStr, uint32* pValue) -{ - char* pRetStr = NULL; - - if (pStr != NULL) { - if (pValue != NULL) { - *pValue = strtoul(pStr, NULL, 0); - } - - pRetStr = strchr(pStr, ','); - if (pRetStr) { - pRetStr++; - } - } - - return pRetStr; -} - -// Parse string element to extract 4-tuples of -// comma-separated uint values and allocate and write those values to output array. -// -// Example input string: -// 0x67A0,0x00,0x06,0x28,0x67A1,0x00,0x06,0x28 -// -// Return the number of 4-tuple entries successfully parsed. -static uint32 parse4TupleValues(const char* element, uint32*& values) -{ - // Early out if something else (e.g oglPanel) has already set this value - if (values != NULL) { - return 0; - } - - const size_t strSize = strlen(element); - - if (0 == strSize) { - return 0; - } - - char * const str = (char*)alloca(sizeof(char) * (strSize + 1)); - char * pCurStr = str; - uint32 curDevID; - uint32 curRevID; - uint32 curClientID; - uint32 curValue; - uint32 numTuples = 0; - - // Find size - memcpy(str, element, strSize); - str[strSize] = '\0'; - while (pCurStr != NULL) { - pCurStr = getNextValue(pCurStr, &curDevID); - if (pCurStr != NULL) { - pCurStr = getNextValue(pCurStr, &curRevID); - if (pCurStr != NULL) { - pCurStr = getNextValue(pCurStr, &curClientID); - if (pCurStr != NULL) { - pCurStr = getNextValue(pCurStr, &curValue); - numTuples ++; - } - } - } - } - - if (numTuples == 0) { - return 0; - } - - // Allocate - values = (uint32*)osMemAlloc(numTuples * sizeof(uint32) * 4); - if (!values) { - return 0; - } - - // Copy - uint32 * pCurValue = values; - pCurStr = str; - memcpy(str, element, strSize); - str[strSize] = '\0'; - while (pCurStr != NULL) { - pCurStr = getNextValue(pCurStr, pCurValue); - pCurValue++; - pCurStr = getNextValue(pCurStr, pCurValue); - pCurValue++; - pCurStr = getNextValue(pCurStr, pCurValue); - pCurValue++; - pCurStr = getNextValue(pCurStr, pCurValue); - pCurValue++; - } - - return numTuples; -} - -static void -CALGSLDevice::parsePowerParam(const char* element, gslRuntimeConfigUint32Value& pwrCount, gslRuntimeConfigUint32pValue& pwrPointer) -{ - uint32 count = 0; - uint32* values = NULL; - count = parse4TupleValues(element, values); - if (0 != count) { - pwrCount.hasValue = true; - pwrCount.value = count; - pwrPointer.hasValue = true; - pwrPointer.value = values; - } -} - -bool -CALGSLDevice::open(uint32 gpuIndex, OpenParams& openData) -{ - gslDeviceOps_ = new amd::Monitor("GSL Device Ops Lock", true); - if (NULL == gslDeviceOps_) { - return false; - } - - unsigned int chainIndex = 0; -#ifdef ATI_OS_WIN - m_gpuIndex = gpuIndex; - m_usePerVPUAdapterModel = true; - m_PerformLazyDeviceInit = true; -#else - void * nativeHandle; - gslDeviceMode deviceMode; - gsAdaptor::getDeviceInitData(gpuIndex, &deviceMode, &chainIndex, &nativeHandle); - - m_nativeDisplayHandle = nativeHandle; - m_deviceMode = deviceMode; -#endif - m_chainIndex = chainIndex; - m_vpuMask = 1 << chainIndex; - - // - // CALBE is required to explicitly manage multiple opens and closes - // assert on the condition for correct usage - // - assert(m_adp == 0); - - memset(&m_dcfg, 0, sizeof(m_dcfg)); - - extern void getConfigFromFile(gslStaticRuntimeConfig &scfg, gslDynamicRuntimeConfig &dcfg); - getConfigFromFile(m_scfg, m_dcfg); - - m_scfg.UsePerVPUAdapterModel.hasValue = true; - m_scfg.UsePerVPUAdapterModel.value = m_usePerVPUAdapterModel; - - m_scfg.DX10SamplerResources.hasValue = true; - m_scfg.DX10SamplerResources.value = true; - - m_scfg.vpuMask.hasValue = true; - m_scfg.vpuMask.value = m_vpuMask; - - m_scfg.bEnableHighPerformanceState.hasValue = true; - m_scfg.bEnableHighPerformanceState.value = openData.enableHighPerformanceState; - - m_scfg.bEnableReusableMemCache.hasValue = true; - m_scfg.bEnableReusableMemCache.value = false; - - parsePowerParam(openData.sclkThreshold, m_scfg.sclkActivityThresholdCount, m_scfg.sclkActivityThresholdPtr); - parsePowerParam(openData.downHysteresis, m_scfg.sclkDownHysteresisCount, m_scfg.sclkDownHysteresisPtr); - parsePowerParam(openData.upHysteresis, m_scfg.sclkUpHysteresisCount, m_scfg.sclkUpHysteresisPtr); - parsePowerParam(openData.powerLimit, m_scfg.packagePowerLimitCount, m_scfg.packagePowerLimitPtr); - parsePowerParam(openData.mclkThreshold, m_scfg.mclkActivityThresholdCount, m_scfg.mclkActivityThresholdPtr); - parsePowerParam(openData.mclkUpHyst, m_scfg.mclkUpHysteresisCount, m_scfg.mclkUpHysteresisPtr); - parsePowerParam(openData.mclkDownHyst, m_scfg.mclkDownHysteresisCount, m_scfg.mclkDownHysteresisPtr); - - m_dcfg.disableMarkUsedInCmdBuf.hasValue = true; - m_dcfg.disableMarkUsedInCmdBuf.value = false; - - // Enable immediate memory release - m_dcfg.immediateMemoryRelease.hasValue = true; - m_dcfg.immediateMemoryRelease.value = true; - - m_dcfg.bEnableSvm.hasValue = true; - m_dcfg.bEnableSvm.value = openData.reportAsOCL12Device ? false : OPENCL_MAJOR >= 2; - - m_dcfg.bEnableFlatAddressing.hasValue = true; -#if defined(ATI_BITS_32) && defined(ATI_OS_LINUX) - m_dcfg.bEnableFlatAddressing.value = false; -#else - m_dcfg.bEnableFlatAddressing.value = openData.reportAsOCL12Device ? false : (OPENCL_MAJOR >= 2); -#endif - - if (GPU_ENABLE_HW_DEBUG) { - m_dcfg.nPatchDumpLevel.hasValue = true; - m_dcfg.nPatchDumpLevel.value |= NPATCHDUMPLEVEL_BITS_HW_DEBUG; - } - - //we can use environment variable CAL_ENABLE_ASYNC_DMA to force dma on or off when we need it - char *s = NULL; - if((s = getenv("CAL_ENABLE_ASYNC_DMA"))) - { - m_dcfg.drmdmaMode.hasValue = true; - m_dcfg.drmdmaMode.value = (atoi(s) == 0) ? GSL_CONFIG_DRMDMA_MODE_FORCE_OFF : GSL_CONFIG_DRMDMA_MODE_DEFAULT; - } - - // Use GPU_USE_SYNC_OBJECTS to force syncobject on or off when we need it - m_dcfg.syncObjectMode.hasValue = true; - m_dcfg.syncObjectMode.value = (GPU_USE_SYNC_OBJECTS) ? - GSL_CONFIG_SYNCOBJECT_MODE_ON : GSL_CONFIG_SYNCOBJECT_MODE_OFF; - - // Use OCL_SET_SVM_SIZE to set SVM size we need - m_dcfg.ndevSVMSize.hasValue = true; - m_dcfg.ndevSVMSize.value = OCL_SET_SVM_SIZE; - - // Use GPU_IFH_MODE to test with IFH mode enabled - m_dcfg.DropFlush.hasValue = true; - m_dcfg.DropFlush.value = (GPU_IFH_MODE == 1); - - // Enable TC compatible htile mode. It's HW feature for VI+ and controlled in HWL. - // Depth interop doesn't support TC compatible htile mode, but OCL needs correct tiling setup. - m_dcfg.bEnableTCCompatibleHtile.hasValue = true; - m_dcfg.bEnableTCCompatibleHtile.value = true; - - int32 asic_id = 0; - if (!SetupAdapter(asic_id)) - { - return false; - } - - // Disable gfx10+ ASICs in GSL - if (asic_id >= GSL_ATIASIC_ID_NAVI10) - { - return false; - } - - if (!SetupContext(asic_id)) - { - return false; - } - - if (m_PerformLazyDeviceInit) - { - // close the adaptor - gsAdaptor::closeAdaptor(m_adp); - m_adp = 0; - } - else - { - PerformFullInitialization(); - } - - return true; -} - -void -CALGSLDevice::close() -{ - m_fullInitialized = false; - if (m_cs != NULL) - { - m_cs->Flush(); - } - - if (m_dstDRMDMAMem) - { - resFree(m_dstDRMDMAMem); - m_dstDRMDMAMem = NULL ; - } - if (m_srcDRMDMAMem) - { - resFree(m_srcDRMDMAMem); - m_srcDRMDMAMem = NULL ; - } - - if (m_cs != NULL) - { - m_cs->destroyTextureResource(m_textureResource); - m_cs->destroySampler(m_textureSampler); - m_cs->destroyQuery(m_mapQuery); - m_cs->destroyQuery(m_mapDMAQuery); - - m_cs->setRenderState(0); - m_cs->destroyRenderState(m_rs); - m_cs->destroySubAllocDesc(); - m_rs = 0; - - m_adp->deleteContext(m_cs); - - m_cs = 0; - } - - if (m_adp != NULL) - { - gsAdaptor::closeAdaptor(m_adp); - m_adp = 0; - } -} - -void -CALGSLDevice::PerformAdapterInitialization(bool ValidateOnly) -{ - // Win10 initialization is more exhaustive. - // @ToDo Check if Win7 can be simplified as well - // If we end up creating a paging fence on Win10 in IOL the slave may not go idle - PerformAdapterInitialization_int(ValidateOnly && m_initLite); -} - -void CALGSLDevice::CloseInitializedAdapter(bool ValidateOnly) -{ - // @ToDo Check if Win7 can be simplified as well - // The adapter shouldnt be destroyed if its created when bindExternalDevice is called - // during context creation - if (m_initLite && ValidateOnly && !m_fullInitialized) - { - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - // close the adaptor - gsAdaptor::closeAdaptor(m_adp); - m_adp = 0; - } -} - -void -CALGSLDevice::PerformFullInitialization() const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - CALGSLDevice* mutable_this = const_cast(this); - mutable_this->PerformFullInitialization_int(); -} - -bool -CALGSLDevice::SetupAdapter(int32 &asic_id) -{ -#ifdef ATI_OS_WIN - m_initLite = true; -#endif - - PerformAdapterInitialization_int(m_initLite); - - if (m_adp == 0) - { - return false; - } - - asic_id = m_adp->getAsicID(); - - if ((asic_id < GSL_ATIASIC_ID_TAHITI_P)) - { - LogPrintfInfo("Unsupported legacy ASIC(%d) found!\n", asic_id); - // close the adaptor - gsAdaptor::closeAdaptor(m_adp); - m_adp = 0; - return false; - } - - bool hasDmaEngine = m_adp->findDMAEngine(); - bool hasComputeEngine = m_adp->findComputeEngine(); - - m_canDMA = hasDmaEngine; - - m_adp->queryAvailableEngines(&m_nEngines, m_engines); - - //Disable DRMDMA on CFX mode for linux on all GPUs. -#ifdef ATI_OS_LINUX - if (m_adp->getNumLinkedVPUs() > 1) - { - m_canDMA = ATIGL_FALSE; - } -#endif - - //The sDMA L2T reading invalid address bug is fixed starting from Vega10, - //and page-fault is not enabled for pre-VI, so we need to workaround - //the bug for ASICs in between. - if (asic_id < GSL_ATIASIC_ID_GREENLAND && - asic_id >= GSL_ATIASIC_ID_BONAIRE_M) - { - m_isSDMAL2TConstrained = true; - } - else - { - m_isSDMAL2TConstrained = false; - } - - if (asic_id < GSL_ATIASIC_ID_TAHITI_P) - { - m_computeRing = false; - } - else - { - m_computeRing = true; - } - - if (!flagIsDefault(GPU_NUM_COMPUTE_RINGS)) - { - m_computeRing = (GPU_NUM_COMPUTE_RINGS != 0); - } - - if ((!flagIsDefault(GPU_SELECT_COMPUTE_RINGS_ID)) && (m_computeRing)) - { - gslEngineID engineID; - engineID = static_cast(GPU_SELECT_COMPUTE_RINGS_ID + GSL_ENGINEID_COMPUTE0); - if ((engineID >= GSL_ENGINEID_COMPUTE0) && (engineID <= GSL_ENGINEID_COMPUTE7)) - { - for (uint i = 0; i < m_nEngines; ++i) { - if (m_engines[i].id == engineID){ - m_isComputeRingIDForced = true; - m_forcedComputeEngineID = engineID; - break; - } - } - } - } - - if (m_computeRing && !hasComputeEngine) - { - return false; - } - - return true; -} - -bool -CALGSLDevice::SetupContext(int32 &asic_id) -{ - gsl::gsCtx* temp_cs = m_adp->createComputeContext(m_computeRing ? (m_isComputeRingIDForced ? m_forcedComputeEngineID : - getFirstAvailableComputeEngineID()) : GSL_ENGINEID_3DCOMPUTE0, - m_canDMA ? GSL_ENGINEID_DRMDMA0 : GSL_ENGINEID_INVALID, m_initLite); - temp_cs->getMainSubCtx()->setVPUMask(m_vpuMask); - - m_maxtexturesize = temp_cs->getMaxTextureSize(); - - switch (asic_id) - { - case GSL_ATIASIC_ID_TAHITI_P: - m_target = CAL_TARGET_TAHITI; - m_elfmachine = ED_ATI_CAL_MACHINE_TAHITI_ISA; - break; - case GSL_ATIASIC_ID_PITCAIRN_PM: - m_target = CAL_TARGET_PITCAIRN; - m_elfmachine = ED_ATI_CAL_MACHINE_PITCAIRN_ISA; - break; - case GSL_ATIASIC_ID_CAPEVERDE_M: - m_target = CAL_TARGET_CAPEVERDE; - m_elfmachine = ED_ATI_CAL_MACHINE_CAPEVERDE_ISA; - break; - case GSL_ATIASIC_ID_OLAND_M: - m_target = CAL_TARGET_OLAND; - m_elfmachine = ED_ATI_CAL_MACHINE_OLAND_ISA; - break; - case GSL_ATIASIC_ID_HAINAN_M: - m_target = CAL_TARGET_HAINAN; - m_elfmachine = ED_ATI_CAL_MACHINE_HAINAN_ISA; - break; - case GSL_ATIASIC_ID_BONAIRE_M: - m_target = CAL_TARGET_BONAIRE; - m_elfmachine = ED_ATI_CAL_MACHINE_BONAIRE_ISA; - break; - case GSL_ATIASIC_ID_SPECTRE: - m_target = CAL_TARGET_SPECTRE; - m_elfmachine = ED_ATI_CAL_MACHINE_SPECTRE_ISA; - break; - case GSL_ATIASIC_ID_SPOOKY: - m_target = CAL_TARGET_SPOOKY; - m_elfmachine = ED_ATI_CAL_MACHINE_SPOOKY_ISA; - break; - case GSL_ATIASIC_ID_KALINDI: - m_target = CAL_TARGET_KALINDI; - m_elfmachine = ED_ATI_CAL_MACHINE_KALINDI_ISA; - break; - case GSL_ATIASIC_ID_HAWAII_P: - m_target = CAL_TARGET_HAWAII; - m_elfmachine = ED_ATI_CAL_MACHINE_HAWAII_ISA; - break; - case GSL_ATIASIC_ID_ICELAND_M: - m_target = CAL_TARGET_ICELAND; - m_elfmachine = ED_ATI_CAL_MACHINE_ICELAND_ISA; - break; - case GSL_ATIASIC_ID_TONGA_P: - m_target = CAL_TARGET_TONGA; - m_elfmachine = ED_ATI_CAL_MACHINE_TONGA_ISA; - break; - case GSL_ATIASIC_ID_GODAVARI: - m_target = CAL_TARGET_GODAVARI; - m_elfmachine = ED_ATI_CAL_MACHINE_GODAVARI_ISA; - break; - case GSL_ATIASIC_ID_FIJI_P: - m_target = CAL_TARGET_FIJI; - m_elfmachine = ED_ATI_CAL_MACHINE_FIJI_ISA; - break; - case GSL_ATIASIC_ID_CARRIZO: - m_target = CAL_TARGET_CARRIZO; - m_elfmachine = ED_ATI_CAL_MACHINE_CARRIZO_ISA; - break; - case GSL_ATIASIC_ID_ELLESMERE: - m_target = CAL_TARGET_ELLESMERE; - m_elfmachine = ED_ATI_CAL_MACHINE_ELLESMERE_ISA; - break; - case GSL_ATIASIC_ID_BAFFIN: - m_target = CAL_TARGET_BAFFIN; - m_elfmachine = ED_ATI_CAL_MACHINE_BAFFIN_ISA; - break; - case GSL_ATIASIC_ID_GREENLAND: - m_target = CAL_TARGET_GREENLAND; - m_elfmachine = ED_ATI_CAL_MACHINE_GREENLAND_ISA; - break; - case GSL_ATIASIC_ID_STONEY: - m_target = CAL_TARGET_STONEY; - m_elfmachine = ED_ATI_CAL_MACHINE_STONEY_ISA; - break; - case GSL_ATIASIC_ID_LEXA: - m_target = CAL_TARGET_LEXA; - m_elfmachine = ED_ATI_CAL_MACHINE_LEXA_ISA; - break; - case GSL_ATIASIC_ID_RAVEN: - m_target = CAL_TARGET_RAVEN; - m_elfmachine = ED_ATI_CAL_MACHINE_RAVEN_ISA; - break; - case GSL_ATIASIC_ID_RAVEN2: - m_target = CAL_TARGET_RAVEN2; - m_elfmachine = ED_ATI_CAL_MACHINE_RAVEN2_ISA; - break; - case GSL_ATIASIC_ID_RENOIR: - m_target = CAL_TARGET_RENOIR; - m_elfmachine = ED_ATI_CAL_MACHINE_RENOIR_ISA; - break; - case GSL_ATIASIC_ID_POLARIS22: - m_target = CAL_TARGET_POLARIS22; - m_elfmachine = ED_ATI_CAL_MACHINE_POLARIS22_ISA; - break; - case GSL_ATIASIC_ID_VEGA12: - m_target = CAL_TARGET_VEGA12; - m_elfmachine = ED_ATI_CAL_MACHINE_VEGA12_ISA; - break; - case GSL_ATIASIC_ID_VEGA20: - m_target = CAL_TARGET_VEGA20; - m_elfmachine = ED_ATI_CAL_MACHINE_VEGA20_ISA; - break; - default: - // 6XX is not supported - m_adp->deleteContext(temp_cs); - gsAdaptor::closeAdaptor(m_adp); - m_adp = 0; - - assert(0); - return false; - } - - //cache device details - getAttribs_int(temp_cs); - temp_cs->getMemInfo(&m_memInfo, GSL_MEMINFO_BASIC); - - assert(temp_cs->getVMMode()); - - m_adp->deleteContext(temp_cs); - - return true; -} - -void -CALGSLDevice::PerformAdapterInitialization_int(bool initLite) -{ - if (m_adp == 0) - { - if (m_usePerVPUAdapterModel) - { - m_adp = gsAdaptor::openAdaptorByIndex(m_gpuIndex, &m_scfg, &m_dcfg, initLite); - } - else - { - m_adp = gsAdaptor::openAdaptor(m_nativeDisplayHandle, m_chainIndex, &m_scfg, &m_dcfg); - } - - assert(m_adp != 0); - } -} - -void -CALGSLDevice::PerformFullInitialization_int() -{ - m_fullInitialized = true; - if (m_adp == 0) - { - PerformAdapterInitialization_int(false); - } - - if (m_cs == 0) - { - m_cs = m_adp->createComputeContext(m_computeRing ? (m_isComputeRingIDForced ? m_forcedComputeEngineID : - getFirstAvailableComputeEngineID()) : GSL_ENGINEID_3DCOMPUTE0, - m_canDMA ? GSL_ENGINEID_DRMDMA0 : GSL_ENGINEID_INVALID, false); - m_cs->getMainSubCtx()->setVPUMask(m_vpuMask); - - // - // Check if the command stream has a DMA connection and allow DMA if there - // is a connection and we can actually DMA - // - bool dmaConnection = m_cs->getDrmDma0Ctx() && m_cs->getDrmDma0Ctx()->ioInfo.iolConnection; - m_allowDMA = (dmaConnection && m_canDMA); - - m_rs = m_cs->createRenderState(); - m_cs->setRenderState(m_rs); - - m_cs->Flush(); - - m_cs->createSubAllocDesc(); - - m_mapQuery = m_cs->createQuery(GSL_SYNC_ATI); - m_mapDMAQuery = m_cs->createQuery(GSL_DRMDMA_SYNC_ATI); - - // Allocate 1x1 FART and Vid memory for DMA flush - CALresourceDesc desc; - memset(&desc, 0, sizeof(CALresourceDesc)); - desc.type = GSL_MOA_MEMORY_AGP; - desc.size.width = 1; - desc.size.height = 1; - desc.format = CM_SURF_FMT_R32F; - desc.channelOrder = GSL_CHANNEL_ORDER_R; - desc.dimension = GSL_MOA_TEXTURE_2D; - m_srcDRMDMAMem = resAlloc(&desc); - - desc.type = GSL_MOA_MEMORY_CARD_EXT_NONEXT; - m_dstDRMDMAMem = resAlloc(&desc); - - m_cs->setDMAFlushBuf(m_srcDRMDMAMem, m_dstDRMDMAMem, 4 /*size of CM_SURF_FMT_R32F*/); - - m_PerformLazyDeviceInit = false; - - m_textureResource = m_cs->createTextureResource(); - m_textureSampler = m_cs->createSampler(); - - // This is a temporary w/a for a CP uCode bug in HWS mode. - // due to this bug, CP uCode loops through a RUNLIST unless - // there is a submission on all queues in HWS mode. - // To force CP uCode to exit the loop, the below code creates - // temporary contexts and submits a packet (via setRenderState) - // to each available compute engines except the first one which - // already had submission in the above code. - // @todo: remove this code once the bug is fixed in CP uCode - if (m_adp->isHWSSupported()) { - gslEngineID usedComputeEngineID = m_isComputeRingIDForced ? - m_forcedComputeEngineID : - getFirstAvailableComputeEngineID(); - for (uint i = 0; i < m_nEngines; ++i) { - if (m_engines[i].id >= GSL_ENGINEID_COMPUTE0 && - m_engines[i].id <= GSL_ENGINEID_COMPUTE7 && - m_engines[i].id != usedComputeEngineID) { - gsl::gsCtx* cs_temp; - gslRenderState rs_temp; - - cs_temp = m_adp->createComputeContext(m_engines[i].id, - m_canDMA ? GSL_ENGINEID_DRMDMA0 : GSL_ENGINEID_INVALID, false); - - cs_temp->getMainSubCtx()->setVPUMask(m_vpuMask); - rs_temp = cs_temp->createRenderState(); - cs_temp->setRenderState(rs_temp); - cs_temp->Flush(); - cs_temp->setRenderState(0); - cs_temp->destroyRenderState(rs_temp); - m_adp->deleteContext(cs_temp); - } - } - } - } -} - -void -Wait(gsl::gsCtx* cs, gslQueryTarget target, gslQueryObject object) -{ - uint64 param; - - uint32 mask = (target == GSL_DRMDMA_SYNC_ATI) ? GSL_ENGINE_MASK(GSL_ENGINEID_DRMDMA0) | GSL_ENGINE_MASK(GSL_ENGINEID_DRMDMA1) : GSL_ENGINEMASK_ALL_BUT_UVD_VCE; - - object->BeginQuery(cs, target, 0, mask); - object->EndQuery(cs, 0); - object->GetResult(cs, ¶m); - - assert(param == 1); -} - -static bool -CALGSLDevice::ResolveAperture(const gslMemObjectAttribTiling tiling) const -{ - // Don't ask for aperture if the tiling is linear. - if ((GSL_MOA_TILING_LINEAR == tiling) || - (GSL_MOA_TILING_LINEAR_GENERAL == tiling)) - { - return false; - } - - // Use aperture. - return true; -} - -gslMemObject -CALGSLDevice::resAlloc(const CALresourceDesc* desc) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - assert(m_cs != 0); - gslMemObject mem = 0; - uint32 flags = desc->flags; - - gslMemObjectAttribs attribs( - GSL_MOA_TEXTURE_1D, // type - GSL_MOA_MEMORY_CARD_EXT_NONEXT, // location XXX - (flags & CAL_RESALLOC_GLOBAL_BUFFER) ? GSL_MOA_TILING_LINEAR : g_CALBETiling_Tiled, // tiling - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_DERIVED, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED // alloc_type - ); - - attribs.location = desc->type; - attribs.vaBase = desc->vaBase; - attribs.section = desc->section; - attribs.isAllocSVM = desc->isAllocSVM; - attribs.isAllocExecute = desc->isAllocExecute; - attribs.minAlignment = desc->minAlignment; - - //!@note GSL asserts with tiled 1D images of any type. - if ((desc->dimension == GSL_MOA_BUFFER) || - (desc->dimension == GSL_MOA_TEXTURE_1D) || - (desc->dimension == GSL_MOA_TEXTURE_1D_ARRAY) || - (desc->dimension == GSL_MOA_TEXTURE_BUFFER)) - { - attribs.tiling = GSL_MOA_TILING_LINEAR; - } - - if (desc->type == GSL_MOA_MEMORY_SYSTEM) - { - // CPU addres and size for pinning - attribs.cpu_address = desc->systemMemory; - attribs.size = desc->systemMemorySize; - - if ((desc->size.width % 64) == 0) - { - attribs.tiling = GSL_MOA_TILING_LINEAR; - } - else - { - // Use linear general if width isn't aligned - attribs.tiling = GSL_MOA_TILING_LINEAR_GENERAL; - } - } - else if (desc->type == GSL_MOA_MEMORY_CARD_EXTERNAL_PHYSICAL) - { - attribs.cpu_address = (void*)desc->busAddress; - } - - // Don't ask for aperture if the tiling is linear. - attribs.useAperture = ResolveAperture(attribs.tiling); - - attribs.channelOrder = desc->channelOrder; - attribs.type = desc->dimension; - if (desc->mipLevels > 1) { - attribs.levels = desc->mipLevels; - attribs.mipmap = true; - } - switch (desc->dimension) - { - case GSL_MOA_BUFFER: - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - break; - case GSL_MOA_TEXTURE_1D: - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - break; - case GSL_MOA_TEXTURE_2D: - mem = m_cs->createMemObject2D(desc->format, desc->size.width, (uint32)desc->size.height, &attribs); - break; - case GSL_MOA_TEXTURE_3D: - mem = m_cs->createMemObject3D(desc->format, desc->size.width, - (uint32)desc->size.height, (uint32)desc->size.depth, &attribs); - break; - case GSL_MOA_TEXTURE_BUFFER: - attribs.type = GSL_MOA_TEXTURE_BUFFER; - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - break; - case GSL_MOA_TEXTURE_1D_ARRAY: - mem = m_cs->createMemObject3D(desc->format, desc->size.width, - 1, (uint32)desc->size.height, &attribs); - break; - case GSL_MOA_TEXTURE_2D_ARRAY: - mem = m_cs->createMemObject3D(desc->format, desc->size.width, - (uint32)desc->size.height, (uint32)desc->size.depth, &attribs); - break; - default: - break; - } - -#ifdef ATI_OS_WIN - if ((desc->section == GSL_SECTION_SVM || desc->section == GSL_SECTION_SVM_ATOMICS) && mem == NULL) { - //svm allocation failure, try one more time after wait. - Wait(m_cs, GSL_SYNC_ATI, m_mapQuery); - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - } -#endif - - return mem; -} - -gslMemObject -CALGSLDevice::resAllocView(gslMemObject res, gslResource3D size, size_t offset, cmSurfFmt format, - gslChannelOrder channelOrder, gslMemObjectAttribType resType, uint32 level, uint32 layer, uint32 flags, - uint64 bytePitch) const -{ - assert(m_cs != 0); - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - gslMemObjectAttribs attribs( - GSL_MOA_TEXTURE_2D, // type. Filled in below based on the base type. - GSL_MOA_MEMORY_ALIAS, // location. Filled in below based on the base location. - GSL_MOA_TILING_LINEAR, // tiling. Filled in below based on the flags passed in. - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_DERIVED, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED // alloc_type - ); - attribs.bytePitch = bytePitch; - attribs.section = res->getAttribs().section; - attribs.isAllocSVM = res->getAttribs().isAllocSVM; - attribs.isAllocExecute = res->getAttribs().isAllocExecute; - - // Need to get the alignment info from hwl. - // Not sure hwl is correct though. Linear aligned 256b, tiled 8kb according to the address library. - uint32 alignment; - switch (flags & ~CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER) - { - case CAL_RESALLOCSLICEVIEW_LINEAR_ALIGNED: - alignment = 256; - attribs.tiling = GSL_MOA_TILING_LINEAR; - break; - case CAL_RESALLOCSLICEVIEW_LINEAR_UNALIGNED: - alignment = 1; - attribs.tiling = GSL_MOA_TILING_LINEAR_GENERAL; - break; - default: - alignment = 8192; - // GSL asserts if this tiled mode is differnt from the original surface. - // (For example, original is GSL_MOA_TILING_MACRO and the new one is GSL_MOA_TILING_TILED) - // Use the original mode for view allocation. - attribs.tiling = res->getAttribs().tiling; - if (attribs.tiling == GSL_MOA_TILING_LINEAR || attribs.tiling == GSL_MOA_TILING_LINEAR_GENERAL) - { - alignment = 256; - } - break; - }; - - // Check any alignment restrictions. - uint64 resPitch = res->getPitch(); - cmSurfFmt baseFormat = res->getFormat(); - uint32 elementSize = cmGetSurfElementSize(static_cast(baseFormat)); - uint64 offsetInBytes = static_cast(offset) * elementSize; - if (offsetInBytes % alignment) - { - return 0; //offset doesn't match alignment requirements. - } - if (attribs.bytePitch == (uint64)-1) - { - attribs.bytePitch = resPitch * elementSize; - } - - // alias has same location as the base resource. - attribs.type = res->getAttribs().type; - attribs.location = res->getAttribs().location; - attribs.displayable = res->getAttribs().displayable; - attribs.channelOrder = channelOrder; - - gslMemObject mo = NULL, levelobject = res; - - bool levelLayer = false; - if (flags & CAL_RESALLOCSLICEVIEW_LEVEL) - { - const gsSubImageParam levelParam(level); - levelobject = m_cs->createSubMemObject(res, GSL_LEVEL, levelParam); - attribs.bytePitch = static_cast(levelobject->getPitch()) * - (levelobject->getBitsPerElement() / 8); - levelLayer = true; - } - if (flags & CAL_RESALLOCSLICEVIEW_LAYER) - { - const gsSubImageParam layerParam(layer); - mo = m_cs->createSubMemObject(levelobject, GSL_LAYER, layerParam); - if (levelobject != res) - { - m_cs->destroyMemObject(levelobject); - } - levelobject = mo; - levelLayer = true; - } - - if (levelLayer) { - // If level/layer object was created, then don't need an extra view - return levelobject; - } - - attribs.type = resType; - switch (resType) - { - case GSL_MOA_BUFFER: - mo = m_cs->createOffsetMemObject1D(levelobject, offsetInBytes, format, - size.width, &attribs); - break; - case GSL_MOA_TEXTURE_1D: - mo = m_cs->createOffsetMemObject1D(levelobject, offsetInBytes, format, - size.width, &attribs); - break; - case GSL_MOA_TEXTURE_2D: - mo = m_cs->createOffsetMemObject2D(levelobject, offsetInBytes, format, - size.width, (uint32)size.height, &attribs); - break; - case GSL_MOA_TEXTURE_3D: - mo = m_cs->createOffsetMemObject3D(levelobject, offsetInBytes, format, - size.width, (uint32)size.height, (uint32)size.depth, &attribs); - break; - case GSL_MOA_TEXTURE_BUFFER: - mo = m_cs->createOffsetMemObject1D(levelobject, offsetInBytes, format, - size.width, &attribs); - break; - case GSL_MOA_TEXTURE_1D_ARRAY: - mo = m_cs->createOffsetMemObject3D(levelobject, offsetInBytes, format, - size.width, 1, (uint32)size.height, &attribs); - break; - case GSL_MOA_TEXTURE_2D_ARRAY: - mo = m_cs->createOffsetMemObject3D(levelobject, offsetInBytes, format, - size.width, (uint32)size.height, (uint32)size.depth, &attribs); - break; - default: - break; - } - - if (levelobject != res) - { - m_cs->destroyMemObject(levelobject); - } - - return mo; -} - -struct GSLDeviceMemMap -{ - gslMemObject mem; - uint32 flags; -}; - -void* -CALGSLDevice::resMapLocal(size_t& pitch, - gslMemObject mem, - gslMapAccessType flags) -{ - // No map really necessary if IOMMUv2 is being used, return the surface address directly - // as CPU can write to it for Linear tiled surfaces only - if (mem->getAttribs().isAllocSVM && mem->getAttribs().tiling <= GSL_MOA_TILING_LINEAR) - { - return (void*)mem->getImage(0)->surf.addr.getAddress(); - } - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - void* pPtr = NULL; - gslMemObjectAttribLocation location = mem->getAttribs().location; - - if ((location == GSL_MOA_MEMORY_CARD_LOCKABLE) || !m_allowDMA) - { - // direct lock - if (location == GSL_MOA_MEMORY_CARD_LOCKABLE) { - // Get tiling mode and resolve the aperture settings. - bool useAperture = ResolveAperture(mem->getAttribs().tiling); - - pPtr = mem->map(m_cs, GSL_MAP_NOSYNC, GSL_GPU_0, false, useAperture); - } - else - { - pPtr = mem->map(m_cs, flags, GSL_GPU_0, true, false); - } - - if (pPtr == NULL) - { - return NULL; - } - - // obtain the pitch of the buffer - pitch = static_cast(mem->getPitch()); - } - else - { - // Allocate map structure for the unmap call - GSLDeviceMemMap* memMap = static_cast(malloc(sizeof(GSLDeviceMemMap))); - - if (memMap == NULL) - { - return NULL; - } - - uint64 width = mem->getRectWidth(); - intp height = mem->getRectHeight(); - cmSurfFmt format = mem->getFormat(); - - gslMemObjectAttribType dstType = mem->getAttribs().type; - - gslMemObjectAttribs attribsDest( - dstType, // type - GSL_MOA_MEMORY_REMOTE_CACHEABLE, // location - GSL_MOA_TILING_LINEAR, // tiling - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_DERIVED, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED // alloc_type - ); - - attribsDest.channelOrder = mem->getAttribs().channelOrder; - - // Create the target destination buffer - memMap->mem = m_cs->createMemObject2D(format, width, (uint32)height, &attribsDest); - - if (memMap->mem == NULL) - { - attribsDest.location = GSL_MOA_MEMORY_AGP; - memMap->mem = m_cs->createMemObject2D(format, width, (uint32)height, &attribsDest); - if (memMap->mem == NULL) - { - free(memMap); - return NULL; - } - } - - // set the pointer to it as the return buffer - pPtr = memMap->mem->map(m_cs, GSL_MAP_NOSYNC, GSL_GPU_0, false, false); - if (pPtr == 0) - { - m_cs->destroyMemObject(memMap->mem); - free(memMap); - return NULL; - } - - // obtain the pitch of the temporary buffer - pitch = static_cast(memMap->mem->getPitch()); - - // For write only cases, we don't care about the data - if (flags != GSL_MAP_WRITE_ONLY) - { - uint64 surfaceSize = memMap->mem->getSurfaceSize(); - uint64 dstSize = mem->getSurfaceSize(); - - surfaceSize = (surfaceSize > dstSize) ? dstSize : surfaceSize; - - //! @todo Workaround strange GSL/CMM-QS behavior. OCL doesn't require a sync, - //! because resource isn't busy on the CAL device. However without sync there are less CBs available - //! Conformanace multidevice test will create around 60 queues, instead of 70 - uint32 mode = (IS_LINUX) ? GSL_SYNCUPLOAD_SYNC_WAIT | GSL_SYNCUPLOAD_SYNC_START : 0; - m_cs->DMACopy(mem, 0, memMap->mem, 0, surfaceSize, mode, NULL); - - Wait(m_cs, GSL_DRMDMA_SYNC_ATI, m_mapDMAQuery); - } - - m_hack.insert(std::pair(mem, (intp) memMap)); - memMap->flags = flags; - } - - return pPtr; -} - -void -CALGSLDevice::resUnmapLocal(gslMemObject mem) -{ - // No unmap necessary with IOMMUv2 as map operation directly returned the base surface System VA - // which CPU can write to it for Linear tiled surfaces only - if (mem->getAttribs().isAllocSVM && mem->getAttribs().tiling <= GSL_MOA_TILING_LINEAR) - { - return; - } - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - // Find the pairing - Hack::iterator iter = m_hack.find(mem); - if (iter == m_hack.end()) - { - // We didn't find a pair, then it's a direct map - mem->unmap(m_cs); - //! @todo: GSL doesn't wait for CB on unmap, - //! thus the data isn't really avaiable to all engines - if (mem->getAttribs().location != GSL_MOA_MEMORY_CARD_LOCKABLE) - { - Wait(m_cs, GSL_SYNC_ATI, m_mapQuery); - } - return; - } - - GSLDeviceMemMap* memMap = static_cast(iter->second); - m_hack.erase(iter); - - memMap->mem->unmap(m_cs); - - if (memMap->flags != GSL_MAP_READ_ONLY) - { - uint64 surfaceSize = memMap->mem->getSurfaceSize(); - uint64 dstSize = mem->getSurfaceSize(); - - surfaceSize = (surfaceSize > dstSize) ? dstSize : surfaceSize; - - //! @todo Workaround strange GSL/CMM-QS behavior. OCL doesn't require a sync, - //! because resource isn't busy on the CAL device. However without sync there are less CBs available - //! Conformanace multidevice test will create around 60 queues, instead of 70 - uint32 mode = (IS_LINUX) ? GSL_SYNCUPLOAD_SYNC_WAIT | GSL_SYNCUPLOAD_SYNC_START : 0; - m_cs->DMACopy(memMap->mem, 0, mem, 0, surfaceSize, mode, NULL); - - Wait(m_cs, GSL_DRMDMA_SYNC_ATI, m_mapDMAQuery); - } - m_cs->destroyMemObject(memMap->mem); - - delete memMap; -} - -gslMemObject -CALGSLDevice::resGetHeap(size_t size) const -{ - assert(m_cs != 0); - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - gslMemObjectAttribs attribs( - GSL_MOA_VIRTUAL_HEAP, // type - GSL_MOA_MEMORY_SYSTEM, // location - GSL_MOA_TILING_LINEAR, // tiling - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_DERIVED, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED, // alloc_type - 0, // channel_order - 0 // size of cpu_address - ); - - gslMemObject rval = m_cs->createMemObject1D(CM_SURF_FMT_R32I, size, &attribs); - - return rval; -} - -void* -CALGSLDevice::resMapRemote( - size_t& pitch, - gslMemObject mem, - gslMapAccessType flags) const -{ - // No map really necessary if IOMMUv2 is being used, return the surface address directly - // as CPU can write to it for Linear tiled surfaces only - if (mem->getAttribs().isAllocSVM && mem->getAttribs().tiling <= GSL_MOA_TILING_LINEAR) - { - return (void*)mem->getImage(0)->surf.addr.getAddress(); - } - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - pitch = static_cast(mem->getPitch()); - return mem->map(m_cs, GSL_MAP_NOSYNC, GSL_GPU_0, false, false); -} - -void -CALGSLDevice::resUnmapRemote(gslMemObject mem) const -{ - // No unmap necessary with IOMMUv2 as map operation directly returned the base surface System VA - // which CPU can write to it for Linear tiled surfaces only - if (mem->getAttribs().isAllocSVM && mem->getAttribs().tiling <= GSL_MOA_TILING_LINEAR) - { - return; - } - - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - mem->unmap(m_cs); -} - -#define CPDMA_THRESHOLD 131072 - -CopyType -CALGSLDevice::GetCopyType( - gslMemObject srcMem, - gslMemObject destMem, - size_t* srcOffset, - size_t* destOffset, - bool allowDMA, - uint32 flags, - size_t size, - bool enableCopyRect) const -{ - CopyType type = USE_NONE; - - gslMemObjectAttribTiling srcTiling = srcMem->getAttribs().tiling; - gslMemObjectAttribTiling dstTiling = destMem->getAttribs().tiling; - gslMemObjectAttribType srcType = srcMem->getAttribs().type; - gslMemObjectAttribType dstType = destMem->getAttribs().type; - uint64 srcSize = srcMem->getSurfaceSize(); - - if (size != 0) - { - srcSize = (srcSize > size) ? size : srcSize; - } - - // CPDMA isnt possible for anything other than a 1D_TEXURE or a BUFFER as it does a blind blob copy without regards to padding - bool isCPDMApossible = ((srcTiling == GSL_MOA_TILING_LINEAR) || srcTiling == GSL_MOA_TILING_LINEAR_GENERAL) && - ((dstTiling == GSL_MOA_TILING_LINEAR) || dstTiling == GSL_MOA_TILING_LINEAR_GENERAL) && - (dstType == GSL_MOA_TEXTURE_1D || dstType == GSL_MOA_BUFFER) && - (srcType == dstType); - - if (!allowDMA && !isCPDMApossible) - { - return USE_NONE; - } - - // - // Use CPDMA for transfers < 128KB - // - if (isCPDMApossible && (((flags != CAL_MEMCOPY_ASYNC) && (srcSize <= CPDMA_THRESHOLD) && !enableCopyRect) || - (allowDMA == false)) ) - { - type = USE_CPDMA; - } - // ### Check for Particular kind of DRMDMA here - else if (allowDMA && - (((srcType == GSL_MOA_TEXTURE_2D) && (dstType == GSL_MOA_BUFFER)) || - ((dstType == GSL_MOA_TEXTURE_2D) && (srcType == GSL_MOA_BUFFER)))) - { - if ((srcTiling != GSL_MOA_TILING_LINEAR) && - (dstTiling == GSL_MOA_TILING_LINEAR)) - { - intp bppSrc = srcMem->getBitsPerElement(); - uint64 BytesPerPixel = bppSrc / 8; - uint64 linearBytePitch = size * BytesPerPixel; - - // Make sure linear pitch in bytes is 4 bytes aligned - if (((linearBytePitch % 4) == 0) && - // another DRM restriciton... SI has 4 pixels - (srcOffset[0] % 4 == 0) && - (destOffset[0] % 4 == 0)) - { - // The sDMA T2L cases we need to avoid are when the tiled_x - // is not a multiple of BytesPerPixel. - if (!m_isSDMAL2TConstrained || - (srcOffset[0] % BytesPerPixel == 0)) - { - type = USE_DRMDMA_T2L; - } - } - } - else if ((srcTiling == GSL_MOA_TILING_LINEAR) && - (dstTiling != GSL_MOA_TILING_LINEAR)) - { - intp bppDst = destMem->getBitsPerElement(); - uint64 BytesPerPixel = bppDst / 8; - uint64 linearBytePitch = size * BytesPerPixel; - - // Make sure linear pitch in bytes is 4 bytes aligned - if (((linearBytePitch % 4) == 0) && - // another DRM restriciton... SI has 4 pixels - (destOffset[0] % 4 == 0) && - (srcOffset[0] % 4 == 0)) - { - // The sDMA L2T cases we need to avoid are when the tiled_x - // is not a multiple of BytesPerPixel. - if (!m_isSDMAL2TConstrained || - (destOffset[0] % BytesPerPixel == 0)) - { - type = USE_DRMDMA_L2T; - } - } - } - } - else if (dstType == srcType) - { - type = USE_DRMDMA; - } - - return type; -} - -uint64 -CALGSLDevice::calcScratchBufferSize(uint32 regNum) const -{ - gslProgramTargetEnum target = GSL_COMPUTE_PROGRAM; - - // Determine the scratch size we need to allocate - cmScratchSpaceNeededPerShaderStage scratchSpacePerShaderStage; - memset(&scratchSpacePerShaderStage, 0, sizeof(scratchSpacePerShaderStage)); - uint64 scratchBufferSizes[gslProgramTarget_COUNT]; - memset(scratchBufferSizes, 0, sizeof(scratchBufferSizes)); - uint32 enabledShadersFlag = 0; - - //!@todo should be CM_COMPUTE_SHADER - enabledShadersFlag |= CM_FRAGMENT_SHADER_BIT; - scratchSpacePerShaderStage.scratchSpace[CM_FRAGMENT_SHADER] = regNum; - target = GSL_FRAGMENT_PROGRAM; - - m_cs->CalcAllScratchBufferSizes(enabledShadersFlag, scratchSpacePerShaderStage, - scratchBufferSizes); - - // SWDEV-79308: - // Reduce the total scratch buffer size by a factor of 4, which in effect reducing the - // max. scratch waves from 32 to 8. This will avoid the required total scratch buffer - // size exceeds the available local memory. (Note: the scratch buffer size needs to - // be 64K alignment) - if (scratchBufferSizes[target] > 0) - { - scratchBufferSizes[target] = (scratchBufferSizes[target] >> 2); - - if (scratchBufferSizes[target] == 0) { // assign minimum scratch buffer size of 64K - scratchBufferSizes[target] = 0x10000; - } - } - - return scratchBufferSizes[target]; -} - -static void -CALGSLDevice::convertInputChannelOrder(intp*channelOrder) const -{ - // set default to indicate that we don't want to override the channel order. - // set all order to zero to indicate default. - channelSwizzle chanSwiz = {SWIZZLE_ZERO, - SWIZZLE_ZERO, - SWIZZLE_ZERO, - SWIZZLE_ZERO}; - - switch (*channelOrder) { - case GSL_CHANNEL_ORDER_R: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_ZERO; - chanSwiz.b = SWIZZLE_ZERO; - chanSwiz.a = SWIZZLE_ONE; - break; - - case GSL_CHANNEL_ORDER_A: - chanSwiz.r = SWIZZLE_ZERO; - chanSwiz.g = SWIZZLE_ZERO; - chanSwiz.b = SWIZZLE_ZERO; - chanSwiz.a = SWIZZLE_COMPONENT0; - break; - - case GSL_CHANNEL_ORDER_RG: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_ZERO; - chanSwiz.a = SWIZZLE_ONE; - break; - - case GSL_CHANNEL_ORDER_RA: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_ZERO; - chanSwiz.b = SWIZZLE_ZERO; - chanSwiz.a = SWIZZLE_COMPONENT1; - break; - - case GSL_CHANNEL_ORDER_RGB: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT2; - chanSwiz.a = SWIZZLE_ONE; - break; - - case GSL_CHANNEL_ORDER_RGBA: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT2; - chanSwiz.a = SWIZZLE_COMPONENT3; - break; - - case GSL_CHANNEL_ORDER_ARGB: - chanSwiz.r = SWIZZLE_COMPONENT1; - chanSwiz.g = SWIZZLE_COMPONENT2; - chanSwiz.b = SWIZZLE_COMPONENT3; - chanSwiz.a = SWIZZLE_COMPONENT0; - break; - - case GSL_CHANNEL_ORDER_BGRA: - chanSwiz.r = SWIZZLE_COMPONENT2; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT0; - chanSwiz.a = SWIZZLE_COMPONENT3; - break; - - case GSL_CHANNEL_ORDER_SRGB: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT2; - chanSwiz.a = SWIZZLE_ONE; - break; - - case GSL_CHANNEL_ORDER_SRGBX: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT2; - chanSwiz.a = SWIZZLE_ONE; - break; - - case GSL_CHANNEL_ORDER_SRGBA: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT2; - chanSwiz.a = SWIZZLE_COMPONENT3; - break; - - case GSL_CHANNEL_ORDER_SBGRA: - chanSwiz.r = SWIZZLE_COMPONENT2; - chanSwiz.g = SWIZZLE_COMPONENT1; - chanSwiz.b = SWIZZLE_COMPONENT0; - chanSwiz.a = SWIZZLE_COMPONENT3; - break; - - case GSL_CHANNEL_ORDER_REPLICATE_R: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT0; - chanSwiz.b = SWIZZLE_COMPONENT0; - chanSwiz.a = SWIZZLE_COMPONENT0; - break; - - case GSL_CHANNEL_ORDER_INTENSITY: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT0; - chanSwiz.b = SWIZZLE_COMPONENT0; - chanSwiz.a = SWIZZLE_COMPONENT0; - break; - - case GSL_CHANNEL_ORDER_LUMINANCE: - chanSwiz.r = SWIZZLE_COMPONENT0; - chanSwiz.g = SWIZZLE_COMPONENT0; - chanSwiz.b = SWIZZLE_COMPONENT0; - chanSwiz.a = SWIZZLE_ONE; - break; - default: assert(0); break; - }; - - *channelOrder = *(uint32 *)&chanSwiz; -} - -void -CALGSLDevice::fillImageHwState(gslMemObject mem, void* hwState, uint32 hwStateSize) const -{ - amd::ScopedLock k(gslDeviceOps()); - intp channelOrder = mem->getAttribs().channelOrder; - convertInputChannelOrder(&channelOrder); - m_textureResource->updateDepthTextureParam(mem); - m_textureResource->getTextureSrd(m_cs, mem, reinterpret_cast(&channelOrder), - hwState, hwStateSize); -} - -void -CALGSLDevice::fillSamplerHwState(bool unnorm, uint32 min, uint32 mag, uint32 addr, - float minLod, float maxLod, void* hwState, uint32 hwStateSize) const -{ - amd::ScopedLock k(gslDeviceOps()); - m_textureSampler->setUnnormalizedMode(m_cs, unnorm); - m_textureSampler->setMinFilter(m_cs, static_cast(min)); - m_textureSampler->setMagFilter(m_cs, static_cast(mag)); - m_textureSampler->setWrap(m_cs, GSL_TEXTURE_WRAP_S, static_cast(addr)); - m_textureSampler->setWrap(m_cs, GSL_TEXTURE_WRAP_T, static_cast(addr)); - m_textureSampler->setWrap(m_cs, GSL_TEXTURE_WRAP_R, static_cast(addr)); - m_textureSampler->setMinLOD(m_cs, static_cast(minLod)); - m_textureSampler->setMaxLOD(m_cs, static_cast(maxLod)); - - m_textureSampler->getSamplerSrd(m_cs, hwState, hwStateSize); -} -bool -CALGSLDevice::gslSetClockMode(GSLClockModeInfo * clockModeInfo) -{ - bool result = false; - const void* requestClockInfo = reinterpret_cast(clockModeInfo); - uint32 uReturn = m_adp->requestClockModeInfo((void*)requestClockInfo); - if(uReturn == GSL_SETCLOCK_SUCCESS || uReturn == GSL_SETCLOCK_QUERY_ONLY) - { - result = true; - } - return result; -} diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.h deleted file mode 100644 index 903bca95bc..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDevice.h +++ /dev/null @@ -1,258 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __GSLDevice_h__ -#define __GSLDevice_h__ - -#include "backend.h" -#include "atitypes.h" -#include "gsl_types.h" -#include "gsl_config.h" -#include "thread/monitor.hpp" -#include "gsl_types_internal.h" - -#ifdef ATI_OS_LINUX -typedef unsigned int IDirect3DDevice9; -typedef unsigned int IDirect3DSurface9; -typedef unsigned int IDirect3DQuery9; -typedef unsigned int RECT; -#else -#undef APIENTRY -#include -#endif - -#include - -namespace gsl -{ - class gsAdaptor; -}; - -typedef enum -{ - USE_NONE, - USE_CPDMA, - USE_DRMDMA, - USE_DRMDMA_L2T, - USE_DRMDMA_T2L, -} CopyType; - -class CALGSLDevice -{ -public: - struct GLResAssociate { - void* GLContext; //(IN) handle to HGLRC or GLXContext - void* GLdeviceContext; //(IN) a handle to device context - uint name; //(IN) gl identifier of the object - uint type; // (IN) type of the interop object . - uint flags; // (IN) flags assigned to 'GLResource' struct - void* mbResHandle; // (OUT) Internal GL driver handle for the resource - gslMemObject mem_base; // (OUT) Base memory object for the resource - gslMemObject memObject; //(OUT) Alias gsl memory object for the resource - gslMemObject fMaskObject; //(OUT) gsl memobject of the an MSAA resource F-mask. - }; - - struct OpenParams { - bool enableHighPerformanceState; - bool reportAsOCL12Device; - const char* sclkThreshold; - const char* downHysteresis; - const char* upHysteresis; - const char* powerLimit; - const char* mclkThreshold; - const char* mclkUpHyst; - const char* mclkDownHyst; - }; - - CALGSLDevice(); - ~CALGSLDevice(); - - bool open(uint32 gpuIndex, OpenParams& openData); - void close(); - - gslMemObject resAlloc(const CALresourceDesc* desc) const; - void* resMapLocal(size_t& pitch, gslMemObject mem, gslMapAccessType flags); - void resUnmapLocal(gslMemObject mem); - - void resFree(gslMemObject mem) const; - void* resMapRemote(size_t& pitch, gslMemObject mem, gslMapAccessType flags) const; - void resUnmapRemote(gslMemObject mem) const; - - gslMemObject resGetHeap(size_t size) const; - gslMemObject resAllocView(gslMemObject res, gslResource3D size, - size_t offset, cmSurfFmt format, gslChannelOrder channelOrder, - gslMemObjectAttribType resType, uint32 level, uint32 layer, - uint32 flags, uint64 bytePitch = (uint64)-1) const; - - bool associateD3D11Device(void* d3d11Device); //void* is of type ID3D11Device* - bool associateD3D10Device(void* d3d10Device); //void* is of type ID3D10Device* - bool associateD3D9Device(void* d3d9Device); //void* is of type IDirect3DDevice9* - - gslMemObject resMapD3DResource( - const CALresourceDesc* desc, uint64 sharedhandle, bool displayable) const; - - bool glAssociate(CALvoid *GLplatformContext, CALvoid* GLdeviceContext); - bool glDissociate(CALvoid *GLplatformContext, CALvoid* GLdeviceContext); - //! @brief This function is called once for every interop resource on the first clEnqeueuAcquireGL. - bool resGLAssociate(GLResAssociate & resData) const; - //! @brief This function is called once for every interop resource on resource destruction. - bool resGLFree (CALvoid* GLplatformContext, - CALvoid* GLdeviceContext, gslMemObject mem, gslMemObject mem_base, - CALvoid* mbResHandle, CALuint type) const; - //! @brief Decompresses depth/MSAA surfaces.This function is called on every 'clEnqeueuAcquireGLObject'. - bool resGLAcquire( CALvoid* GLplatformContext, CALvoid* mbResHandle, CALuint type) const; - //! @brief This function is called on every 'clEnqeueuReleaseGLObject'. - bool resGLRelease(CALvoid* GLplatformContext, CALvoid* mbResHandle, CALuint type) const; - - gsl::gsAdaptor* getNative() const; - CALuint getElfMachine() const { return m_elfmachine; }; - uint32 getGpuIndex() const { return m_gpuIndex; }; - - uint32 getMaxTextureSize() const; - const CALdeviceattribs& getAttribs() const { return m_attribs; } - const gslMemInfo& getMemInfo() const { return m_memInfo; } - - uint32 getVPUMask() const { return m_vpuMask; } - bool canDMA() const { return m_canDMA; } - gslMemObject m_srcDRMDMAMem, m_dstDRMDMAMem; // memory object of flush buffer, used for DRMDMA flush - - void PerformAdapterInitialization(bool ValidateOnly); - void PerformFullInitialization() const; - void CloseInitializedAdapter(bool ValidateOnly); - - CopyType GetCopyType(gslMemObject srcMem, gslMemObject destMem, size_t* srcOffset, - size_t* destOffset, bool allowDMA, uint32 flags, size_t size, bool enableCopyRect) const; - - uint64 calcScratchBufferSize(uint32 regNum) const; - - amd::Monitor& gslDeviceOps() const { return *gslDeviceOps_; } - - void fillImageHwState(gslMemObject mem, void* hwState, uint32 hwStateSize) const; - - void fillSamplerHwState(bool unnorm, uint32 min, uint32 mag, uint32 addr, - float minLod, float maxLod, void* hwState, uint32 hwStateSize) const; - - gslSamplerObject txSampler() const { return m_textureSampler; } - - void convertInputChannelOrder(intp *channelOrder) const; - - gsl::gsCtx* gslCtx() const { return m_cs; } - - bool isComputeRingIDForced() const { return m_isComputeRingIDForced; } - gslEngineID getforcedComputeEngineID() const { return m_forcedComputeEngineID; } - - gslEngineID getFirstAvailableComputeEngineID() const { return static_cast( - m_adp->findFirstAvailableComputeEngineID()); } - - virtual bool gslSetClockMode(GSLClockModeInfo * clockModeInfo); -protected: - // - /// channel order enumerants - // - //channelSwizzleMode and channelSwizzle match the hwl equivalent hwtxSwizzleMode and hwtxUnitSwizzle in hwl_tx_if.h. - enum channelSwizzleMode { - SWIZZLE_COMPONENT0, ///< Select Component0 - SWIZZLE_COMPONENT1, ///< Select Component1 - SWIZZLE_COMPONENT2, ///< Select Component2 - SWIZZLE_COMPONENT3, ///< Select Component3 - SWIZZLE_ZERO, ///< Select Zero - SWIZZLE_ONE, ///< Select One - }; - - // - /// channel order swizzle type - // - typedef struct channelSwizzleRec - { - channelSwizzleMode r : 8; ///< Red channel of texture - channelSwizzleMode g : 8; ///< Green channel of texture - channelSwizzleMode b : 8; ///< Blue channel of texture - channelSwizzleMode a : 8; ///< Alpha channel of texture - } channelSwizzle; - - uint m_nEngines; - gslEngineDescriptor m_engines[GSL_ENGINEID_MAX]; - -private: - gsl::gsAdaptor* m_adp; - gsl::gsCtx* m_cs; - gslRenderState m_rs; - CALtarget m_target; - CALuint m_elfmachine; - uint32 m_vpuMask; - uint32 m_chainIndex; - int32 m_maxtexturesize; - uint32 m_gpuIndex; - void* m_nativeDisplayHandle; - - gslDeviceModeEnum m_deviceMode; - - typedef std::map Hack; - Hack m_hack; - gslQueryObject m_mapQuery; - gslQueryObject m_mapDMAQuery; - - gslStaticRuntimeConfig m_scfg; - gslDynamicRuntimeConfig m_dcfg; - - //GL Extension specific - bool initGLInteropPrivateExt(CALvoid* GLplatformContext, CALvoid* GLdeviceContext) const; - bool glCanInterop(CALvoid* GLplatformContext, CALvoid* GLdeviceContext); - - bool PerformDMACopy(gslMemObject srcMem, gslMemObject destMem, cmSurfFmt format, CALuint flags, bool isHwDebug = false); - void Initialize(void); - - bool SetupAdapter(int32 &asic_id); - bool SetupContext(int32 &asic_id); - void PerformAdapterInitialization_int(bool initLite); - void PerformFullInitialization_int(); - - void getAttribs_int(gsl::gsCtx* cs); - bool ResolveAperture(const gslMemObjectAttribTiling tiling) const; - - void parsePowerParam(const char* element, gslRuntimeConfigUint32Value& pwrCount, gslRuntimeConfigUint32pValue& pwrPointer); - - CALdeviceattribs m_attribs; - gslMemInfo m_memInfo; - gslTextureResourceObject m_textureResource; - gslSamplerObject m_textureSampler; - gslEngineID m_forcedComputeEngineID; - - union { - struct { - uint m_canDMA : 1; - uint m_allowDMA : 1; - uint m_computeRing : 1; - uint m_usePerVPUAdapterModel : 1; - uint m_PerformLazyDeviceInit : 1; - uint m_isComputeRingIDForced : 1; - uint m_isSDMAL2TConstrained : 1; - uint m_initLite : 1; - uint m_fullInitialized : 1; - }; - - uint m_flags; - }; - - amd::Monitor* gslDeviceOps_; //!< Lock to serialize GSL device -}; - -#endif // __GSLDevice_h__ - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D10.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D10.cpp deleted file mode 100644 index e7462f12d9..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D10.cpp +++ /dev/null @@ -1,251 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "GSLDevice.h" - -#if defined(ATI_OS_WIN) - -#include - -/************************************************************************************************************** -* Note: ideally the DXX extension interfaces should be mapped from the DXX perforce branch. -* This means CAL client spec will need to change to include headers directly from the DXX perforce tree. -* However, CAL only cares about the DXX OpenCL extension interface class. The spec cannot change -* without notification. So it is safe to use a local copy of the relevant DXX extension interface classes. -**************************************************************************************************************/ -#include "DxxOpenCLInteropExt.h" - -static bool -queryD3D10DeviceGPUMask(ID3D10Device* pd3d10Device, UINT* pd3d10DeviceGPUMask) -{ - HMODULE hDLL = NULL; - IAmdDxExt* pExt = NULL; - IAmdDxExtCLInterop* pCLExt = NULL; - PFNAmdDxExtCreate AmdDxExtCreate; - HRESULT hr = S_OK; - - // Get a handle to the DXX DLL with extension API support -#if defined _WIN64 - static const CHAR dxxModuleName[13] = "atidxx64.dll"; -#else - static const CHAR dxxModuleName[13] = "atidxx32.dll"; -#endif - - hDLL = GetModuleHandle(dxxModuleName); - - if (hDLL == NULL) - { - hr = E_FAIL; - } - - // Get the exported AmdDxExtCreate() function pointer - if (SUCCEEDED(hr)) - { - AmdDxExtCreate = reinterpret_cast(GetProcAddress(hDLL, "AmdDxExtCreate")); - if (AmdDxExtCreate == NULL) - { - hr = E_FAIL; - } - } - - // Create the extension object - if (SUCCEEDED(hr)) - { - hr = AmdDxExtCreate(pd3d10Device, &pExt); - } - - // Get the extension version information - if (SUCCEEDED(hr)) - { - AmdDxExtVersion extVersion; - hr = pExt->GetVersion(&extVersion); - - if (extVersion.majorVersion == 0) - { - hr = E_FAIL; - } - } - - // Get the OpenCL Interop interface - if (SUCCEEDED(hr)) - { - pCLExt = static_cast(pExt->GetExtInterface(AmdDxExtCLInteropID)); - if (pCLExt != NULL) - { - // Get the GPU mask using the CL Interop extension. - pCLExt->QueryInteropGpuMask(pd3d10DeviceGPUMask); - } - else - { - hr = E_FAIL; - } - } - - if (pCLExt != NULL) - { - pCLExt->Release(); - } - - if (pExt != NULL) - { - pExt->Release(); - } - - return (SUCCEEDED(hr)); -} - -bool -CALGSLDevice::associateD3D10Device(void* d3d10Device) -{ - bool canInteroperate = false; - - LUID calDevAdapterLuid = {0, 0}; - UINT calDevChainBitMask = 0; - UINT d3d10DeviceGPUMask = 0; - - ID3D10Device* pd3d10Device = static_cast(d3d10Device); - - IDXGIDevice* pDXGIDevice; - pd3d10Device->QueryInterface(__uuidof(IDXGIDevice), (void **)&pDXGIDevice); - - IDXGIAdapter* pDXGIAdapter; - pDXGIDevice->GetAdapter(&pDXGIAdapter); - - DXGI_ADAPTER_DESC adapterDesc; - pDXGIAdapter->GetDesc(&adapterDesc); - - // match the adapter - if (m_adp->getMVPUinfo(&calDevAdapterLuid, &calDevChainBitMask)) - { - canInteroperate = ((calDevAdapterLuid.HighPart == adapterDesc.AdapterLuid.HighPart) && - (calDevAdapterLuid.LowPart == adapterDesc.AdapterLuid.LowPart)); - } - - // match the chain ID - if (canInteroperate) - { - if (queryD3D10DeviceGPUMask(pd3d10Device, &d3d10DeviceGPUMask)) - { - canInteroperate = (calDevChainBitMask & d3d10DeviceGPUMask) != 0; - } - else - { - // special handling for Intel iGPU + AMD dGPU in LDA mode (only occurs on a PX platform) where - // the D3D10Device object is created on the Intel iGPU and passed to AMD dGPU (secondary) to interoperate. - if (calDevChainBitMask > 1) - { - canInteroperate = false; - } - } - } - - pDXGIDevice->Release(); - pDXGIAdapter->Release(); - - return canInteroperate; -} - -gslMemObject -CALGSLDevice::resMapD3DResource(const CALresourceDesc* desc, uint64 sharedhandle, bool displayable) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps_); - - gslMemObject mem = NULL; - - gslMemObjectAttribs attribs( - GSL_MOA_TEXTURE_2D, // type - GSL_MOA_MEMORY_ALIAS, // location - GSL_MOA_TILING_TILED, // tiling - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_DERIVED, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED // alloc_type - ); - - HANDLE h = (HANDLE)sharedhandle; - attribs.cpu_address = h; - attribs.alias_swizzle = 0; - attribs.channelOrder = desc->channelOrder; - attribs.type = desc->dimension; - - switch (desc->dimension) - { - case GSL_MOA_BUFFER: - attribs.tiling = GSL_MOA_TILING_LINEAR; - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - break; - case GSL_MOA_TEXTURE_1D: - attribs.tiling = GSL_MOA_TILING_LINEAR; - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - break; - case GSL_MOA_TEXTURE_2D: - { - uint32 height = (uint32)desc->size.height; - if (displayable) - { - attribs.displayable = GSL_MOA_DISPLAYABLE_YES; - } - mem = m_cs->createMemObject2D(desc->format, desc->size.width, height, &attribs); - } - break; - case GSL_MOA_TEXTURE_3D: - mem = m_cs->createMemObject3D(desc->format, desc->size.width, - (uint32)desc->size.height, (uint32)desc->size.depth, &attribs); - break; - case GSL_MOA_TEXTURE_BUFFER: - attribs.type = GSL_MOA_TEXTURE_BUFFER; - mem = m_cs->createMemObject1D(desc->format, desc->size.width, &attribs); - break; - case GSL_MOA_TEXTURE_1D_ARRAY: - mem = m_cs->createMemObject3D(desc->format, desc->size.width, - 1, (uint32)desc->size.height, &attribs); - break; - case GSL_MOA_TEXTURE_2D_ARRAY: - mem = m_cs->createMemObject3D(desc->format, desc->size.width, - (uint32)desc->size.height, (uint32)desc->size.depth, &attribs); - break; - default: - break; - } - - return mem; -} - -#else // !ATI_OS_WIN - -bool -CALGSLDevice::associateD3D10Device(void* d3d10Device) -{ - return false; -} - -gslMemObject -CALGSLDevice::resMapD3DResource(const CALresourceDesc* desc, uint64 sharedhandle, bool displayable) const -{ - return 0; -} - -#endif // !ATI_OS_WIN diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D11.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D11.cpp deleted file mode 100644 index de95930a67..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D11.cpp +++ /dev/null @@ -1,174 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "GSLDevice.h" - -#if defined(ATI_OS_WIN) - -#include - -/************************************************************************************************************** -* Note: ideally the DXX extension interfaces should be mapped from the DXX perforce branch. -* This means CAL client spec will need to change to include headers directly from the DXX perforce tree. -* However, CAL only cares about the DXX OpenCL extension interface class. The spec cannot change -* without notification. So it is safe to use a local copy of the relevant DXX extension interface classes. -**************************************************************************************************************/ -#include "DxxOpenCLInteropExt.h" - -static bool -queryD3D11DeviceGPUMask(ID3D11Device* pd3d11Device, UINT* pd3d11DeviceGPUMask) -{ - HMODULE hDLL = NULL; - IAmdDxExt* pExt = NULL; - IAmdDxExtCLInterop* pCLExt = NULL; - PFNAmdDxExtCreate11 AmdDxExtCreate11; - HRESULT hr = S_OK; - - // Get a handle to the DXX DLL with extension API support -#if defined _WIN64 - static const CHAR dxxModuleName[13] = "atidxx64.dll"; -#else - static const CHAR dxxModuleName[13] = "atidxx32.dll"; -#endif - - hDLL = GetModuleHandle(dxxModuleName); - - if (hDLL == NULL) - { - hr = E_FAIL; - } - - // Get the exported AmdDxExtCreate() function pointer - if (SUCCEEDED(hr)) - { - AmdDxExtCreate11 = reinterpret_cast(GetProcAddress(hDLL, "AmdDxExtCreate11")); - if (AmdDxExtCreate11 == NULL) - { - hr = E_FAIL; - } - } - - // Create the extension object - if (SUCCEEDED(hr)) - { - hr = AmdDxExtCreate11(pd3d11Device, &pExt); - } - - // Get the extension version information - if (SUCCEEDED(hr)) - { - AmdDxExtVersion extVersion; - hr = pExt->GetVersion(&extVersion); - - if (extVersion.majorVersion == 0) - { - hr = E_FAIL; - } - } - - // Get the OpenCL Interop interface - if (SUCCEEDED(hr)) - { - pCLExt = static_cast(pExt->GetExtInterface(AmdDxExtCLInteropID)); - if (pCLExt != NULL) - { - // Get the GPU mask using the CL Interop extension. - pCLExt->QueryInteropGpuMask(pd3d11DeviceGPUMask); - } - else - { - hr = E_FAIL; - } - } - - if (pCLExt != NULL) - { - pCLExt->Release(); - } - - if (pExt != NULL) - { - pExt->Release(); - } - - return (SUCCEEDED(hr)); -} - -bool -CALGSLDevice::associateD3D11Device(void* d3d11Device) -{ - bool canInteroperate = false; - - LUID calDevAdapterLuid = {0, 0}; - UINT calDevChainBitMask = 0; - UINT d3d11DeviceGPUMask = 0; - - ID3D11Device* pd3d11Device = static_cast(d3d11Device); - - IDXGIDevice* pDXGIDevice; - pd3d11Device->QueryInterface(__uuidof(IDXGIDevice), (void **)&pDXGIDevice); - - IDXGIAdapter* pDXGIAdapter; - pDXGIDevice->GetAdapter(&pDXGIAdapter); - - DXGI_ADAPTER_DESC adapterDesc; - pDXGIAdapter->GetDesc(&adapterDesc); - - // match the adapter - if (m_adp->getMVPUinfo(&calDevAdapterLuid, &calDevChainBitMask)) - { - canInteroperate = ((calDevAdapterLuid.HighPart == adapterDesc.AdapterLuid.HighPart) && - (calDevAdapterLuid.LowPart == adapterDesc.AdapterLuid.LowPart)); - } - - // match the chain ID - if (canInteroperate) - { - if (queryD3D11DeviceGPUMask(pd3d11Device, &d3d11DeviceGPUMask)) - { - canInteroperate = (calDevChainBitMask & d3d11DeviceGPUMask) != 0; - } - else - { - // special handling for Intel iGPU + AMD dGPU in LDA mode (only occurs on a PX platform) where - // the D3D11Device object is created on the Intel iGPU and passed to AMD dGPU (secondary) to interoperate. - if (calDevChainBitMask > 1) - { - canInteroperate = false; - } - } - } - - pDXGIDevice->Release(); - pDXGIAdapter->Release(); - - return canInteroperate; -} - -#else // !ATI_OS_WIN - -bool -CALGSLDevice::associateD3D11Device(void* d3d11Device) -{ - return false; -} - -#endif // !ATI_OS_WIN diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D9.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D9.cpp deleted file mode 100644 index 3f4b569d38..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceD3D9.cpp +++ /dev/null @@ -1,77 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "GSLDevice.h" - -#if defined(ATI_OS_WIN) - -#include -#include - -/************************************************************************************************************** -* Note: ideally the DXX extension interfaces should be mapped from the DXX perforce branch. -* This means CAL client spec will need to change to include headers directly from the DXX perforce tree. -* However, CAL only cares about the DXX OpenCL extension interface class. The spec cannot change -* without notification. So it is safe to use a local copy of the relevant DXX extension interface classes. -**************************************************************************************************************/ -#include "DxxOpenCLInteropExt.h" - - -bool -CALGSLDevice::associateD3D9Device(void* d3d9Device) -{ - bool canInteroperate = false; - D3DCAPS9 pCaps; - LUID calDevAdapterLuid = {0, 0}; - UINT calDevChainBitMask = 0; - IDirect3D9* p3d9dev; - LUID d3d9deviceLuid = {0, 0}; - - IDirect3DDevice9* pd3d9Device = static_cast(d3d9Device); - - // Get D3D9 Device caps - pd3d9Device->GetDeviceCaps(&pCaps); - // Get 3D9 Device - pd3d9Device->GetDirect3D(&p3d9dev); - - IDirect3D9Ex* p3d9devEx = static_cast(p3d9dev); - p3d9devEx->GetAdapterLUID(pCaps.AdapterOrdinal, &d3d9deviceLuid); - p3d9dev->Release(); - - // match the adapter - if (m_adp->getMVPUinfo(&calDevAdapterLuid, &calDevChainBitMask)) - { - canInteroperate = ((calDevAdapterLuid.HighPart == d3d9deviceLuid.HighPart) && - (calDevAdapterLuid.LowPart == d3d9deviceLuid.LowPart)); - } - - return canInteroperate; -} - -#else // !ATI_OS_WIN - -bool -CALGSLDevice::associateD3D9Device(void* d3d9Device) -{ - return false; -} - -#endif // !ATI_OS_WIN diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceGL.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceGL.cpp deleted file mode 100644 index 9d57b46e6a..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLDeviceGL.cpp +++ /dev/null @@ -1,1033 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "GSLDevice.h" -#include "component_types.h" -#include "cwddeci.h" -#include -#include "GL/glATIInternal.h" -#ifdef ATI_OS_LINUX -#include -#include -#include "GL/glx.h" -#include "GL/glxext.h" -#include "GL/glXATIPrivate.h" -#else -#include "GL/wglATIPrivate.h" -#endif -#include "memory/MemObject.h" - -typedef struct cmFormatXlateRec{ - cmSurfFmt raw_cmFormat; - cmSurfFmt cal_cmFormat; - gslChannelOrder channelOrder; -} cmFormatXlateParams; - -// relates full range of cm surface formats to those supported by CAL -static constexpr cmFormatXlateParams cmFormatXlateTable [] = { - {CM_SURF_FMT_LUMINANCE8, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE16, CM_SURF_FMT_R16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE16F, CM_SURF_FMT_R16F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE32F, CM_SURF_FMT_R32F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY8, CM_SURF_FMT_INTENSITY8, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_INTENSITY16, CM_SURF_FMT_R16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY16F, CM_SURF_FMT_R16F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY32F, CM_SURF_FMT_R32F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ALPHA8, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ALPHA16, CM_SURF_FMT_R16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ALPHA16F, CM_SURF_FMT_R16F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ALPHA32F, CM_SURF_FMT_R32F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE8_ALPHA8, CM_SURF_FMT_RG8I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_LUMINANCE16_ALPHA16, CM_SURF_FMT_RG16I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_LUMINANCE16F_ALPHA16F, CM_SURF_FMT_RG16F, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_LUMINANCE32F_ALPHA32F, CM_SURF_FMT_RG16F, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_B2_G3_R3, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_B5_G6_R5, CM_SURF_FMT_B5_G6_R5, GSL_CHANNEL_ORDER_RGB}, - {CM_SURF_FMT_BGRX4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGR5_X1, CM_SURF_FMT_BGR5_X1, GSL_CHANNEL_ORDER_RGB}, - {CM_SURF_FMT_BGRX8, CM_SURF_FMT_RGBA8, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGR10_X2, CM_SURF_FMT_BGR10_X2, GSL_CHANNEL_ORDER_RGB}, - {CM_SURF_FMT_BGRX16, CM_SURF_FMT_RGBA16, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGRX16F, CM_SURF_FMT_RGBA16F, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGRX32F, CM_SURF_FMT_RGBA32F, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_RGBX4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGB5_X1, CM_SURF_FMT_BGR5_X1, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX8, CM_SURF_FMT_RGBA8, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGB10_X2, CM_SURF_FMT_RGB10_X2, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX16, CM_SURF_FMT_RGBA16, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX16F, CM_SURF_FMT_RGBA16F, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX32F, CM_SURF_FMT_RGBA32F, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_BGRA4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGR5_A1, CM_SURF_FMT_BGR5_X1, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGRA8, CM_SURF_FMT_RGBA8, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGR10_A2, CM_SURF_FMT_BGR10_X2, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGRA16, CM_SURF_FMT_RGBA16, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGRA16F, CM_SURF_FMT_RGBA16, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_BGRA32F, CM_SURF_FMT_RGBA32F, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_RGBA4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGB5_A1, CM_SURF_FMT_BGR5_X1, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8, CM_SURF_FMT_RGBA8, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGB10_A2, CM_SURF_FMT_BGR10_X2, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA16, CM_SURF_FMT_RGBA16, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA16F, CM_SURF_FMT_RGBA16F, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA32I, CM_SURF_FMT_RGBA32I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA32F, CM_SURF_FMT_RGBA32F, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_DUDV8, CM_SURF_FMT_RG8I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_DXT1, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DXT2_3, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DXT4_5, (cmSurfFmt)00, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ATI1N, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ATI2N, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DEPTH16, CM_SURF_FMT_DEPTH16, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_DEPTH16F, CM_SURF_FMT_R16F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DEPTH24_X8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DEPTH24F_X8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DEPTH24_STEN8, CM_SURF_FMT_DEPTH24_STEN8, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_DEPTH24F_STEN8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DEPTH32F_X24_STEN8, CM_SURF_FMT_DEPTH32F_X24_STEN8, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_DEPTH32F, CM_SURF_FMT_DEPTH32F, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_sR11_sG11_sB10, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sU16, CM_SURF_FMT_sU16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sUV16, CM_SURF_FMT_sUV16, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sUVWQ16, CM_SURF_FMT_sUVWQ16, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RG16, CM_SURF_FMT_RG16, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RG16F, CM_SURF_FMT_RG16F, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RG32F, CM_SURF_FMT_RG32F, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_ABGR4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_A1_BGR5, CM_SURF_FMT_BGR5_X1, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_ABGR8, CM_SURF_FMT_RGBA8, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_A2_BGR10, CM_SURF_FMT_BGR10_X2, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_ABGR16, CM_SURF_FMT_RGBA16, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_ABGR16F, CM_SURF_FMT_RGBA16F, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_ABGR32F, CM_SURF_FMT_RGBA32F, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_DXT1A, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sRGB10_A2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_sR8, CM_SURF_FMT_sR8, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sRG8, CM_SURF_FMT_sRG8, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sR32I, CM_SURF_FMT_sR32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sRG32I, CM_SURF_FMT_sRG32I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sRGBA32I, CM_SURF_FMT_sRGBA32I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_R32I, CM_SURF_FMT_R32I, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_RG32I, CM_SURF_FMT_RG32I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RG8, CM_SURF_FMT_RG8, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sRGBA8, CM_SURF_FMT_sRGBA8, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_R11F_G11F_B10F, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGB9_E5, CM_SURF_FMT_RGBA8, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_LUMINANCE_LATC1, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SIGNED_LUMINANCE_LATC1,(cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_LUMINANCE_ALPHA_LATC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SIGNED_LUMINANCE_ALPHA_LATC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RED_RGTC1, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SIGNED_RED_RGTC1, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RED_GREEN_RGTC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SIGNED_RED_GREEN_RGTC2,(cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_R8, CM_SURF_FMT_INTENSITY8, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_R16, CM_SURF_FMT_R16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_R16F, CM_SURF_FMT_R16F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_R32F, CM_SURF_FMT_R32F, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_R8I, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sR8I, CM_SURF_FMT_sR8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_RG8I, CM_SURF_FMT_RG8I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sRG8I, CM_SURF_FMT_sRG8I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_R16I, CM_SURF_FMT_R16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sR16I, CM_SURF_FMT_sR16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_RG16I, CM_SURF_FMT_RG16I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sRG16I, CM_SURF_FMT_sRG16I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RGBA32UI, CM_SURF_FMT_RGBA32UI, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX32UI, CM_SURF_FMT_RGBA32UI, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_ALPHA32UI, CM_SURF_FMT_R32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY32UI, CM_SURF_FMT_R32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE32UI, CM_SURF_FMT_R32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE_ALPHA32UI, CM_SURF_FMT_RG32I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RGBA16UI, CM_SURF_FMT_RGBA16UI, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX16UI, CM_SURF_FMT_RGBA16UI, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_ALPHA16UI, CM_SURF_FMT_R16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY16UI, CM_SURF_FMT_R16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE16UI, CM_SURF_FMT_R16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE_ALPHA16UI, CM_SURF_FMT_R32I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RGBA8UI, CM_SURF_FMT_RGBA8UI, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX8UI, CM_SURF_FMT_RGBA8UI, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_ALPHA8UI, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY8UI, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE8UI, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE_ALPHA8UI, CM_SURF_FMT_RG8I, GSL_CHANNEL_ORDER_RG}, -#ifndef CM_SURF_FMT_sRGBA32I_EXT_HAS_BEEN_REMOVED - {CM_SURF_FMT_sRGBA32I_EXT, CM_SURF_FMT_sRGBA32I, GSL_CHANNEL_ORDER_RGBA}, -#endif - {CM_SURF_FMT_sRGBX32I, CM_SURF_FMT_sRGBA32I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_sALPHA32I, CM_SURF_FMT_sR32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sINTENSITY32I, CM_SURF_FMT_sR32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sLUMINANCE32I, CM_SURF_FMT_sR32I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sLUMINANCE_ALPHA32I, CM_SURF_FMT_sRG32I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sRGBA16I, CM_SURF_FMT_sRGBA16I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_sRGBX16I, CM_SURF_FMT_sRGBA16I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_sALPHA16I, CM_SURF_FMT_sR16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sINTENSITY16I, CM_SURF_FMT_sR16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sLUMINANCE16I, CM_SURF_FMT_sR16I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sLUMINANCE_ALPHA16I, CM_SURF_FMT_sRG16I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sRGBA8I, CM_SURF_FMT_sRGBA8I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_sRGBX8I, CM_SURF_FMT_sRGBA8I, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_sALPHA8I, CM_SURF_FMT_sR8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sINTENSITY8I, CM_SURF_FMT_sR8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sLUMINANCE8I, CM_SURF_FMT_sR8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_sLUMINANCE_ALPHA8I, CM_SURF_FMT_sRG8I, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_sDXT6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DXT6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_DXT7, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE8_SNORM, CM_SURF_FMT_sR8, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE16_SNORM, CM_SURF_FMT_sU16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY8_SNORM, CM_SURF_FMT_sR8, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_INTENSITY16_SNORM, CM_SURF_FMT_sU16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ALPHA8_SNORM, CM_SURF_FMT_sR8, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_ALPHA16_SNORM, CM_SURF_FMT_sU16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_LUMINANCE_ALPHA8_SNORM,CM_SURF_FMT_sRG8, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_LUMINANCE_ALPHA16_SNORM,CM_SURF_FMT_sUV16, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_R8_SNORM, CM_SURF_FMT_sR8, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_R16_SNORM, CM_SURF_FMT_sU16, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_RG8_SNORM, CM_SURF_FMT_sRG8, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RG16_SNORM, CM_SURF_FMT_sUV16, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RGBX8_SNORM, CM_SURF_FMT_sRGBA8, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBX16_SNORM, CM_SURF_FMT_sUVWQ16, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_SNORM, CM_SURF_FMT_sRGBA8, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA16_SNORM, CM_SURF_FMT_sUVWQ16, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGB8_ETC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGB}, - {CM_SURF_FMT_SRGB8_ETC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGB}, - {CM_SURF_FMT_RGB8_PT_ALPHA1_ETC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGB8_PT_ALPHA1_ETC2, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ETC2_EAC, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGB8_ALPHA8_ETC2_EAC, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_R11_EAC, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_SIGNED_R11_EAC, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_RG11_EAC, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_SIGNED_RG11_EAC, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RG}, - {CM_SURF_FMT_RGBA8_ASTC_4x4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_5x4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_5x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_6x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_6x6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_8x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_8x6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_8x8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_10x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_10x6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_10x8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_10x10, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_12x10, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_RGBA8_ASTC_12x12, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - - {CM_SURF_FMT_SRGBA8_ASTC_4x4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_5x4, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_5x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_6x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_6x6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_8x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_8x6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_8x8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_10x5, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_10x6, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_10x8, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_10x10, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_12x10, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_SRGBA8_ASTC_12x12, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_BGR10_A2UI, (cmSurfFmt)501, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_A2_BGR10UI, (cmSurfFmt)501, GSL_CHANNEL_ORDER_ARGB}, - {CM_SURF_FMT_A2_RGB10UI, (cmSurfFmt)501, GSL_CHANNEL_ORDER_ABGR}, - {CM_SURF_FMT_B5_G6_R5UI, (cmSurfFmt)500, GSL_CHANNEL_ORDER_BGRA}, - {CM_SURF_FMT_R5_G6_B5UI, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_DEPTH32F_X24_STEN8_UNCLAMPED, CM_SURF_FMT_RG32I, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_DEPTH32F_UNCLAMPED, CM_SURF_FMT_R32F, GSL_CHANNEL_ORDER_REPLICATE_R}, - {CM_SURF_FMT_L8_X16_A8_SRGB, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_L8_X24_SRGB, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RGBA}, - {CM_SURF_FMT_STENCIL8, CM_SURF_FMT_R8I, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_L8_SRGB, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_R8_SRGB, (cmSurfFmt)500, GSL_CHANNEL_ORDER_R}, - {CM_SURF_FMT_RG8_SRGB, (cmSurfFmt)500, GSL_CHANNEL_ORDER_RG}, - }; - -FINLINE void -dummyAssertIfCmSurfFmtChanges(void) -{ - // - // Assert if cmSurfFmt defined in ugl/src/include/cmndefs.h changes. - // - COMPILE_TIME_ASSERT(cmSurfFmt_FIRST == CM_SURF_FMT_LUMINANCE8); - COMPILE_TIME_ASSERT( 0 == CM_SURF_FMT_LUMINANCE8); - COMPILE_TIME_ASSERT( 1 == CM_SURF_FMT_LUMINANCE16); - COMPILE_TIME_ASSERT( 2 == CM_SURF_FMT_LUMINANCE16F); - COMPILE_TIME_ASSERT( 3 == CM_SURF_FMT_LUMINANCE32F); - COMPILE_TIME_ASSERT( 4 == CM_SURF_FMT_INTENSITY8); - COMPILE_TIME_ASSERT( 5 == CM_SURF_FMT_INTENSITY16); - COMPILE_TIME_ASSERT( 6 == CM_SURF_FMT_INTENSITY16F); - COMPILE_TIME_ASSERT( 7 == CM_SURF_FMT_INTENSITY32F); - COMPILE_TIME_ASSERT( 8 == CM_SURF_FMT_ALPHA8); - COMPILE_TIME_ASSERT( 9 == CM_SURF_FMT_ALPHA16); - COMPILE_TIME_ASSERT( 10 == CM_SURF_FMT_ALPHA16F); - COMPILE_TIME_ASSERT( 11 == CM_SURF_FMT_ALPHA32F); - COMPILE_TIME_ASSERT( 12 == CM_SURF_FMT_LUMINANCE8_ALPHA8); - COMPILE_TIME_ASSERT( 13 == CM_SURF_FMT_LUMINANCE16_ALPHA16); - COMPILE_TIME_ASSERT( 14 == CM_SURF_FMT_LUMINANCE16F_ALPHA16F); - COMPILE_TIME_ASSERT( 15 == CM_SURF_FMT_LUMINANCE32F_ALPHA32F); - COMPILE_TIME_ASSERT( 16 == CM_SURF_FMT_B2_G3_R3); - COMPILE_TIME_ASSERT( 17 == CM_SURF_FMT_B5_G6_R5); - COMPILE_TIME_ASSERT( 18 == CM_SURF_FMT_BGRX4); - COMPILE_TIME_ASSERT( 19 == CM_SURF_FMT_BGR5_X1); - COMPILE_TIME_ASSERT( 20 == CM_SURF_FMT_BGRX8); - COMPILE_TIME_ASSERT( 21 == CM_SURF_FMT_BGR10_X2); - COMPILE_TIME_ASSERT( 22 == CM_SURF_FMT_BGRX16); - COMPILE_TIME_ASSERT( 23 == CM_SURF_FMT_BGRX16F); - COMPILE_TIME_ASSERT( 24 == CM_SURF_FMT_BGRX32F); - COMPILE_TIME_ASSERT( 25 == CM_SURF_FMT_RGBX4); - COMPILE_TIME_ASSERT( 26 == CM_SURF_FMT_RGB5_X1); - COMPILE_TIME_ASSERT( 27 == CM_SURF_FMT_RGBX8); - COMPILE_TIME_ASSERT( 28 == CM_SURF_FMT_RGB10_X2); - COMPILE_TIME_ASSERT( 29 == CM_SURF_FMT_RGBX16); - COMPILE_TIME_ASSERT( 30 == CM_SURF_FMT_RGBX16F); - COMPILE_TIME_ASSERT( 31 == CM_SURF_FMT_RGBX32F); - COMPILE_TIME_ASSERT( 32 == CM_SURF_FMT_BGRA4); - COMPILE_TIME_ASSERT( 33 == CM_SURF_FMT_BGR5_A1); - COMPILE_TIME_ASSERT( 34 == CM_SURF_FMT_BGRA8); - COMPILE_TIME_ASSERT( 35 == CM_SURF_FMT_BGR10_A2); - COMPILE_TIME_ASSERT( 36 == CM_SURF_FMT_BGRA16); - COMPILE_TIME_ASSERT( 37 == CM_SURF_FMT_BGRA16F); - COMPILE_TIME_ASSERT( 38 == CM_SURF_FMT_BGRA32F); - COMPILE_TIME_ASSERT( 39 == CM_SURF_FMT_RGBA4); - COMPILE_TIME_ASSERT( 40 == CM_SURF_FMT_RGB5_A1); - COMPILE_TIME_ASSERT( 41 == CM_SURF_FMT_RGBA8); - COMPILE_TIME_ASSERT( 42 == CM_SURF_FMT_RGB10_A2); - COMPILE_TIME_ASSERT( 43 == CM_SURF_FMT_RGBA16); - COMPILE_TIME_ASSERT( 44 == CM_SURF_FMT_RGBA16F); - COMPILE_TIME_ASSERT( 45 == CM_SURF_FMT_RGBA32I); - COMPILE_TIME_ASSERT( 46 == CM_SURF_FMT_RGBA32F); - COMPILE_TIME_ASSERT( 47 == CM_SURF_FMT_DUDV8); - COMPILE_TIME_ASSERT( 48 == CM_SURF_FMT_DXT1); - COMPILE_TIME_ASSERT( 49 == CM_SURF_FMT_DXT2_3); - COMPILE_TIME_ASSERT( 50 == CM_SURF_FMT_DXT4_5); - COMPILE_TIME_ASSERT( 51 == CM_SURF_FMT_ATI1N); - COMPILE_TIME_ASSERT( 52 == CM_SURF_FMT_ATI2N); - COMPILE_TIME_ASSERT( 53 == CM_SURF_FMT_DEPTH16); - COMPILE_TIME_ASSERT( 54 == CM_SURF_FMT_DEPTH16F); - COMPILE_TIME_ASSERT( 55 == CM_SURF_FMT_DEPTH24_X8); - COMPILE_TIME_ASSERT( 56 == CM_SURF_FMT_DEPTH24F_X8); - COMPILE_TIME_ASSERT( 57 == CM_SURF_FMT_DEPTH24_STEN8); - COMPILE_TIME_ASSERT( 58 == CM_SURF_FMT_DEPTH24F_STEN8); - COMPILE_TIME_ASSERT( 59 == CM_SURF_FMT_DEPTH32F_X24_STEN8); - COMPILE_TIME_ASSERT( 60 == CM_SURF_FMT_DEPTH32F); - COMPILE_TIME_ASSERT( 61 == CM_SURF_FMT_sR11_sG11_sB10); - COMPILE_TIME_ASSERT( 62 == CM_SURF_FMT_sU16); - COMPILE_TIME_ASSERT( 63 == CM_SURF_FMT_sUV16); - COMPILE_TIME_ASSERT( 64 == CM_SURF_FMT_sUVWQ16); - COMPILE_TIME_ASSERT( 65 == CM_SURF_FMT_RG16); - COMPILE_TIME_ASSERT( 66 == CM_SURF_FMT_RG16F); - COMPILE_TIME_ASSERT( 67 == CM_SURF_FMT_RG32F); - COMPILE_TIME_ASSERT( 68 == CM_SURF_FMT_ABGR4); - COMPILE_TIME_ASSERT( 69 == CM_SURF_FMT_A1_BGR5); - COMPILE_TIME_ASSERT( 70 == CM_SURF_FMT_ABGR8); - COMPILE_TIME_ASSERT( 71 == CM_SURF_FMT_A2_BGR10); - COMPILE_TIME_ASSERT( 72 == CM_SURF_FMT_ABGR16); - COMPILE_TIME_ASSERT( 73 == CM_SURF_FMT_ABGR16F); - COMPILE_TIME_ASSERT( 74 == CM_SURF_FMT_ABGR32F); - COMPILE_TIME_ASSERT( 75 == CM_SURF_FMT_DXT1A); - COMPILE_TIME_ASSERT( 76 == CM_SURF_FMT_sRGB10_A2); - COMPILE_TIME_ASSERT( 77 == CM_SURF_FMT_sR8); - COMPILE_TIME_ASSERT( 78 == CM_SURF_FMT_sRG8); - COMPILE_TIME_ASSERT( 79 == CM_SURF_FMT_sR32I); - COMPILE_TIME_ASSERT( 80 == CM_SURF_FMT_sRG32I); - COMPILE_TIME_ASSERT( 81 == CM_SURF_FMT_sRGBA32I); - COMPILE_TIME_ASSERT( 82 == CM_SURF_FMT_R32I); - COMPILE_TIME_ASSERT( 83 == CM_SURF_FMT_RG32I); - COMPILE_TIME_ASSERT( 84 == CM_SURF_FMT_RG8); - COMPILE_TIME_ASSERT( 85 == CM_SURF_FMT_sRGBA8); - COMPILE_TIME_ASSERT( 86 == CM_SURF_FMT_R11F_G11F_B10F); - COMPILE_TIME_ASSERT( 87 == CM_SURF_FMT_RGB9_E5); - COMPILE_TIME_ASSERT( 88 == CM_SURF_FMT_LUMINANCE_LATC1); - COMPILE_TIME_ASSERT( 89 == CM_SURF_FMT_SIGNED_LUMINANCE_LATC1); - COMPILE_TIME_ASSERT( 90 == CM_SURF_FMT_LUMINANCE_ALPHA_LATC2); - COMPILE_TIME_ASSERT( 91 == CM_SURF_FMT_SIGNED_LUMINANCE_ALPHA_LATC2); - COMPILE_TIME_ASSERT( 92 == CM_SURF_FMT_RED_RGTC1); - COMPILE_TIME_ASSERT( 93 == CM_SURF_FMT_SIGNED_RED_RGTC1); - COMPILE_TIME_ASSERT( 94 == CM_SURF_FMT_RED_GREEN_RGTC2); - COMPILE_TIME_ASSERT( 95 == CM_SURF_FMT_SIGNED_RED_GREEN_RGTC2); - COMPILE_TIME_ASSERT( 96 == CM_SURF_FMT_R8); - COMPILE_TIME_ASSERT( 97 == CM_SURF_FMT_R16); - COMPILE_TIME_ASSERT( 98 == CM_SURF_FMT_R16F); - COMPILE_TIME_ASSERT( 99 == CM_SURF_FMT_R32F); - COMPILE_TIME_ASSERT(100 == CM_SURF_FMT_R8I); - COMPILE_TIME_ASSERT(101 == CM_SURF_FMT_sR8I); - COMPILE_TIME_ASSERT(102 == CM_SURF_FMT_RG8I); - COMPILE_TIME_ASSERT(103 == CM_SURF_FMT_sRG8I); - COMPILE_TIME_ASSERT(104 == CM_SURF_FMT_R16I); - COMPILE_TIME_ASSERT(105 == CM_SURF_FMT_sR16I); - COMPILE_TIME_ASSERT(106 == CM_SURF_FMT_RG16I); - COMPILE_TIME_ASSERT(107 == CM_SURF_FMT_sRG16I); - COMPILE_TIME_ASSERT(108 == CM_SURF_FMT_RGBA32UI); - COMPILE_TIME_ASSERT(109 == CM_SURF_FMT_RGBX32UI); - COMPILE_TIME_ASSERT(110 == CM_SURF_FMT_ALPHA32UI); - COMPILE_TIME_ASSERT(111 == CM_SURF_FMT_INTENSITY32UI); - COMPILE_TIME_ASSERT(112 == CM_SURF_FMT_LUMINANCE32UI); - COMPILE_TIME_ASSERT(113 == CM_SURF_FMT_LUMINANCE_ALPHA32UI); - COMPILE_TIME_ASSERT(114 == CM_SURF_FMT_RGBA16UI); - COMPILE_TIME_ASSERT(115 == CM_SURF_FMT_RGBX16UI); - COMPILE_TIME_ASSERT(116 == CM_SURF_FMT_ALPHA16UI); - COMPILE_TIME_ASSERT(117 == CM_SURF_FMT_INTENSITY16UI); - COMPILE_TIME_ASSERT(118 == CM_SURF_FMT_LUMINANCE16UI); - COMPILE_TIME_ASSERT(119 == CM_SURF_FMT_LUMINANCE_ALPHA16UI); - COMPILE_TIME_ASSERT(120 == CM_SURF_FMT_RGBA8UI); - COMPILE_TIME_ASSERT(121 == CM_SURF_FMT_RGBX8UI); - COMPILE_TIME_ASSERT(122 == CM_SURF_FMT_ALPHA8UI); - COMPILE_TIME_ASSERT(123 == CM_SURF_FMT_INTENSITY8UI); - COMPILE_TIME_ASSERT(124 == CM_SURF_FMT_LUMINANCE8UI); - COMPILE_TIME_ASSERT(125 == CM_SURF_FMT_LUMINANCE_ALPHA8UI); -#ifndef CM_SURF_FMT_sRGBA32I_EXT_HAS_BEEN_REMOVED - COMPILE_TIME_ASSERT(126 == CM_SURF_FMT_sRGBA32I_EXT); - COMPILE_TIME_ASSERT(127 == CM_SURF_FMT_sRGBX32I); - COMPILE_TIME_ASSERT(128 == CM_SURF_FMT_sALPHA32I); - COMPILE_TIME_ASSERT(129 == CM_SURF_FMT_sINTENSITY32I); - COMPILE_TIME_ASSERT(130 == CM_SURF_FMT_sLUMINANCE32I); - COMPILE_TIME_ASSERT(131 == CM_SURF_FMT_sLUMINANCE_ALPHA32I); - COMPILE_TIME_ASSERT(132 == CM_SURF_FMT_sRGBA16I); - COMPILE_TIME_ASSERT(133 == CM_SURF_FMT_sRGBX16I); - COMPILE_TIME_ASSERT(134 == CM_SURF_FMT_sALPHA16I); - COMPILE_TIME_ASSERT(135 == CM_SURF_FMT_sINTENSITY16I); - COMPILE_TIME_ASSERT(136 == CM_SURF_FMT_sLUMINANCE16I); - COMPILE_TIME_ASSERT(137 == CM_SURF_FMT_sLUMINANCE_ALPHA16I); - COMPILE_TIME_ASSERT(138 == CM_SURF_FMT_sRGBA8I); - COMPILE_TIME_ASSERT(139 == CM_SURF_FMT_sRGBX8I); - COMPILE_TIME_ASSERT(140 == CM_SURF_FMT_sALPHA8I); - COMPILE_TIME_ASSERT(141 == CM_SURF_FMT_sINTENSITY8I); - COMPILE_TIME_ASSERT(142 == CM_SURF_FMT_sLUMINANCE8I); - COMPILE_TIME_ASSERT(143 == CM_SURF_FMT_sLUMINANCE_ALPHA8I); - COMPILE_TIME_ASSERT(144 == CM_SURF_FMT_sDXT6); - COMPILE_TIME_ASSERT(145 == CM_SURF_FMT_DXT6); - COMPILE_TIME_ASSERT(146 == CM_SURF_FMT_DXT7); - COMPILE_TIME_ASSERT(147 == CM_SURF_FMT_LUMINANCE8_SNORM); - COMPILE_TIME_ASSERT(148 == CM_SURF_FMT_LUMINANCE16_SNORM); - COMPILE_TIME_ASSERT(149 == CM_SURF_FMT_INTENSITY8_SNORM); - COMPILE_TIME_ASSERT(150 == CM_SURF_FMT_INTENSITY16_SNORM); - COMPILE_TIME_ASSERT(151 == CM_SURF_FMT_ALPHA8_SNORM); - COMPILE_TIME_ASSERT(152 == CM_SURF_FMT_ALPHA16_SNORM); - COMPILE_TIME_ASSERT(153 == CM_SURF_FMT_LUMINANCE_ALPHA8_SNORM); - COMPILE_TIME_ASSERT(154 == CM_SURF_FMT_LUMINANCE_ALPHA16_SNORM); - COMPILE_TIME_ASSERT(155 == CM_SURF_FMT_R8_SNORM); - COMPILE_TIME_ASSERT(156 == CM_SURF_FMT_R16_SNORM); - COMPILE_TIME_ASSERT(157 == CM_SURF_FMT_RG8_SNORM); - COMPILE_TIME_ASSERT(158 == CM_SURF_FMT_RG16_SNORM); - COMPILE_TIME_ASSERT(159 == CM_SURF_FMT_RGBX8_SNORM); - COMPILE_TIME_ASSERT(160 == CM_SURF_FMT_RGBX16_SNORM); - COMPILE_TIME_ASSERT(161 == CM_SURF_FMT_RGBA8_SNORM); - COMPILE_TIME_ASSERT(162 == CM_SURF_FMT_RGBA16_SNORM); - COMPILE_TIME_ASSERT(163 == CM_SURF_FMT_RGB10_A2UI); - COMPILE_TIME_ASSERT(164 == CM_SURF_FMT_RGB32F); - COMPILE_TIME_ASSERT(165 == CM_SURF_FMT_RGB32I); - COMPILE_TIME_ASSERT(166 == CM_SURF_FMT_RGB32UI); - COMPILE_TIME_ASSERT(167 == CM_SURF_FMT_RGBX8_SRGB); - COMPILE_TIME_ASSERT(168 == CM_SURF_FMT_RGBA8_SRGB); - COMPILE_TIME_ASSERT(169 == CM_SURF_FMT_DXT1_SRGB); - COMPILE_TIME_ASSERT(170 == CM_SURF_FMT_DXT1A_SRGB); - COMPILE_TIME_ASSERT(171 == CM_SURF_FMT_DXT2_3_SRGB); - COMPILE_TIME_ASSERT(172 == CM_SURF_FMT_DXT4_5_SRGB); - COMPILE_TIME_ASSERT(173 == CM_SURF_FMT_DXT7_SRGB); - COMPILE_TIME_ASSERT(174 == CM_SURF_FMT_RGB8_ETC2); - COMPILE_TIME_ASSERT(175 == CM_SURF_FMT_SRGB8_ETC2); - COMPILE_TIME_ASSERT(176 == CM_SURF_FMT_RGB8_PT_ALPHA1_ETC2); - COMPILE_TIME_ASSERT(177 == CM_SURF_FMT_SRGB8_PT_ALPHA1_ETC2); - COMPILE_TIME_ASSERT(178 == CM_SURF_FMT_RGBA8_ETC2_EAC); - COMPILE_TIME_ASSERT(179 == CM_SURF_FMT_SRGB8_ALPHA8_ETC2_EAC); - COMPILE_TIME_ASSERT(180 == CM_SURF_FMT_R11_EAC); - COMPILE_TIME_ASSERT(181 == CM_SURF_FMT_SIGNED_R11_EAC); - COMPILE_TIME_ASSERT(182 == CM_SURF_FMT_RG11_EAC); - COMPILE_TIME_ASSERT(183 == CM_SURF_FMT_SIGNED_RG11_EAC); - COMPILE_TIME_ASSERT(184 == CM_SURF_FMT_RGBA8_ASTC_4x4); - COMPILE_TIME_ASSERT(185 == CM_SURF_FMT_RGBA8_ASTC_5x4); - COMPILE_TIME_ASSERT(186 == CM_SURF_FMT_RGBA8_ASTC_5x5); - COMPILE_TIME_ASSERT(187 == CM_SURF_FMT_RGBA8_ASTC_6x5); - COMPILE_TIME_ASSERT(188 == CM_SURF_FMT_RGBA8_ASTC_6x6); - COMPILE_TIME_ASSERT(189 == CM_SURF_FMT_RGBA8_ASTC_8x5); - COMPILE_TIME_ASSERT(190 == CM_SURF_FMT_RGBA8_ASTC_8x6); - COMPILE_TIME_ASSERT(191 == CM_SURF_FMT_RGBA8_ASTC_8x8); - COMPILE_TIME_ASSERT(192 == CM_SURF_FMT_RGBA8_ASTC_10x5); - COMPILE_TIME_ASSERT(193 == CM_SURF_FMT_RGBA8_ASTC_10x6); - COMPILE_TIME_ASSERT(194 == CM_SURF_FMT_RGBA8_ASTC_10x8); - COMPILE_TIME_ASSERT(195 == CM_SURF_FMT_RGBA8_ASTC_10x10); - COMPILE_TIME_ASSERT(196 == CM_SURF_FMT_RGBA8_ASTC_12x10); - COMPILE_TIME_ASSERT(197 == CM_SURF_FMT_RGBA8_ASTC_12x12); - COMPILE_TIME_ASSERT(198 == CM_SURF_FMT_SRGBA8_ASTC_4x4); - COMPILE_TIME_ASSERT(199 == CM_SURF_FMT_SRGBA8_ASTC_5x4); - COMPILE_TIME_ASSERT(200 == CM_SURF_FMT_SRGBA8_ASTC_5x5); - COMPILE_TIME_ASSERT(201 == CM_SURF_FMT_SRGBA8_ASTC_6x5); - COMPILE_TIME_ASSERT(202 == CM_SURF_FMT_SRGBA8_ASTC_6x6); - COMPILE_TIME_ASSERT(203 == CM_SURF_FMT_SRGBA8_ASTC_8x5); - COMPILE_TIME_ASSERT(204 == CM_SURF_FMT_SRGBA8_ASTC_8x6); - COMPILE_TIME_ASSERT(205 == CM_SURF_FMT_SRGBA8_ASTC_8x8); - COMPILE_TIME_ASSERT(206 == CM_SURF_FMT_SRGBA8_ASTC_10x5); - COMPILE_TIME_ASSERT(207 == CM_SURF_FMT_SRGBA8_ASTC_10x6); - COMPILE_TIME_ASSERT(208 == CM_SURF_FMT_SRGBA8_ASTC_10x8); - COMPILE_TIME_ASSERT(209 == CM_SURF_FMT_SRGBA8_ASTC_10x10); - COMPILE_TIME_ASSERT(210 == CM_SURF_FMT_SRGBA8_ASTC_12x10); - COMPILE_TIME_ASSERT(211 == CM_SURF_FMT_SRGBA8_ASTC_12x12); - COMPILE_TIME_ASSERT(212 == CM_SURF_FMT_BGR10_A2UI); - COMPILE_TIME_ASSERT(213 == CM_SURF_FMT_A2_BGR10UI); - COMPILE_TIME_ASSERT(214 == CM_SURF_FMT_A2_RGB10UI); - COMPILE_TIME_ASSERT(215 == CM_SURF_FMT_B5_G6_R5UI); - COMPILE_TIME_ASSERT(216 == CM_SURF_FMT_R5_G6_B5UI); - COMPILE_TIME_ASSERT(217 == CM_SURF_FMT_DEPTH32F_X24_STEN8_UNCLAMPED); - COMPILE_TIME_ASSERT(218 == CM_SURF_FMT_DEPTH32F_UNCLAMPED); - COMPILE_TIME_ASSERT(219 == CM_SURF_FMT_L8_X16_A8_SRGB); - COMPILE_TIME_ASSERT(220 == CM_SURF_FMT_L8_X24_SRGB); - COMPILE_TIME_ASSERT(221 == CM_SURF_FMT_STENCIL8); - COMPILE_TIME_ASSERT(222 == CM_SURF_FMT_L8_SRGB); - COMPILE_TIME_ASSERT(223 == CM_SURF_FMT_R8_SRGB); - COMPILE_TIME_ASSERT(224 == CM_SURF_FMT_RG8_SRGB); -#else - COMPILE_TIME_ASSERT(126 == CM_SURF_FMT_sRGBX32I); - COMPILE_TIME_ASSERT(127 == CM_SURF_FMT_sALPHA32I); - COMPILE_TIME_ASSERT(128 == CM_SURF_FMT_sINTENSITY32I); - COMPILE_TIME_ASSERT(129 == CM_SURF_FMT_sLUMINANCE32I); - COMPILE_TIME_ASSERT(130 == CM_SURF_FMT_sLUMINANCE_ALPHA32I); - COMPILE_TIME_ASSERT(131 == CM_SURF_FMT_sRGBA16I); - COMPILE_TIME_ASSERT(132 == CM_SURF_FMT_sRGBX16I); - COMPILE_TIME_ASSERT(133 == CM_SURF_FMT_sALPHA16I); - COMPILE_TIME_ASSERT(134 == CM_SURF_FMT_sINTENSITY16I); - COMPILE_TIME_ASSERT(135 == CM_SURF_FMT_sLUMINANCE16I); - COMPILE_TIME_ASSERT(136 == CM_SURF_FMT_sLUMINANCE_ALPHA16I); - COMPILE_TIME_ASSERT(137 == CM_SURF_FMT_sRGBA8I); - COMPILE_TIME_ASSERT(138 == CM_SURF_FMT_sRGBX8I); - COMPILE_TIME_ASSERT(139 == CM_SURF_FMT_sALPHA8I); - COMPILE_TIME_ASSERT(140 == CM_SURF_FMT_sINTENSITY8I); - COMPILE_TIME_ASSERT(141 == CM_SURF_FMT_sLUMINANCE8I); - COMPILE_TIME_ASSERT(142 == CM_SURF_FMT_sLUMINANCE_ALPHA8I); - COMPILE_TIME_ASSERT(143 == CM_SURF_FMT_sDXT6); - COMPILE_TIME_ASSERT(144 == CM_SURF_FMT_DXT6); - COMPILE_TIME_ASSERT(145 == CM_SURF_FMT_DXT7); - COMPILE_TIME_ASSERT(146 == CM_SURF_FMT_LUMINANCE8_SNORM); - COMPILE_TIME_ASSERT(147 == CM_SURF_FMT_LUMINANCE16_SNORM); - COMPILE_TIME_ASSERT(148 == CM_SURF_FMT_INTENSITY8_SNORM); - COMPILE_TIME_ASSERT(149 == CM_SURF_FMT_INTENSITY16_SNORM); - COMPILE_TIME_ASSERT(150 == CM_SURF_FMT_ALPHA8_SNORM); - COMPILE_TIME_ASSERT(151 == CM_SURF_FMT_ALPHA16_SNORM); - COMPILE_TIME_ASSERT(152 == CM_SURF_FMT_LUMINANCE_ALPHA8_SNORM); - COMPILE_TIME_ASSERT(153 == CM_SURF_FMT_LUMINANCE_ALPHA16_SNORM); - COMPILE_TIME_ASSERT(154 == CM_SURF_FMT_R8_SNORM); - COMPILE_TIME_ASSERT(155 == CM_SURF_FMT_R16_SNORM); - COMPILE_TIME_ASSERT(156 == CM_SURF_FMT_RG8_SNORM); - COMPILE_TIME_ASSERT(157 == CM_SURF_FMT_RG16_SNORM); - COMPILE_TIME_ASSERT(158 == CM_SURF_FMT_RGBX8_SNORM); - COMPILE_TIME_ASSERT(159 == CM_SURF_FMT_RGBX16_SNORM); - COMPILE_TIME_ASSERT(160 == CM_SURF_FMT_RGBA8_SNORM); - COMPILE_TIME_ASSERT(161 == CM_SURF_FMT_RGBA16_SNORM); - COMPILE_TIME_ASSERT(162 == CM_SURF_FMT_RGB10_A2UI); - COMPILE_TIME_ASSERT(163 == CM_SURF_FMT_RGB32F); - COMPILE_TIME_ASSERT(164 == CM_SURF_FMT_RGB32I); - COMPILE_TIME_ASSERT(165 == CM_SURF_FMT_RGB32UI); - COMPILE_TIME_ASSERT(166 == CM_SURF_FMT_RGBX8_SRGB); - COMPILE_TIME_ASSERT(167 == CM_SURF_FMT_RGBA8_SRGB); - COMPILE_TIME_ASSERT(168 == CM_SURF_FMT_DXT1_SRGB); - COMPILE_TIME_ASSERT(169 == CM_SURF_FMT_DXT1A_SRGB); - COMPILE_TIME_ASSERT(170 == CM_SURF_FMT_DXT2_3_SRGB); - COMPILE_TIME_ASSERT(171 == CM_SURF_FMT_DXT4_5_SRGB); - COMPILE_TIME_ASSERT(172 == CM_SURF_FMT_DXT7_SRGB); - COMPILE_TIME_ASSERT(173 == CM_SURF_FMT_RGB8_ETC2); - COMPILE_TIME_ASSERT(174 == CM_SURF_FMT_SRGB8_ETC2); - COMPILE_TIME_ASSERT(175 == CM_SURF_FMT_RGB8_PT_ALPHA1_ETC2); - COMPILE_TIME_ASSERT(176 == CM_SURF_FMT_SRGB8_PT_ALPHA1_ETC2); - COMPILE_TIME_ASSERT(177 == CM_SURF_FMT_RGBA8_ETC2_EAC); - COMPILE_TIME_ASSERT(178 == CM_SURF_FMT_SRGB8_ALPHA8_ETC2_EAC); - COMPILE_TIME_ASSERT(179 == CM_SURF_FMT_R11_EAC); - COMPILE_TIME_ASSERT(180 == CM_SURF_FMT_SIGNED_R11_EAC); - COMPILE_TIME_ASSERT(181 == CM_SURF_FMT_RG11_EAC); - COMPILE_TIME_ASSERT(182 == CM_SURF_FMT_SIGNED_RG11_EAC); - COMPILE_TIME_ASSERT(183 == CM_SURF_FMT_RGBA8_ASTC_4x4); - COMPILE_TIME_ASSERT(184 == CM_SURF_FMT_RGBA8_ASTC_5x4); - COMPILE_TIME_ASSERT(185 == CM_SURF_FMT_RGBA8_ASTC_5x5); - COMPILE_TIME_ASSERT(186 == CM_SURF_FMT_RGBA8_ASTC_6x5); - COMPILE_TIME_ASSERT(187 == CM_SURF_FMT_RGBA8_ASTC_6x6); - COMPILE_TIME_ASSERT(188 == CM_SURF_FMT_RGBA8_ASTC_8x5); - COMPILE_TIME_ASSERT(189 == CM_SURF_FMT_RGBA8_ASTC_8x6); - COMPILE_TIME_ASSERT(190 == CM_SURF_FMT_RGBA8_ASTC_8x8); - COMPILE_TIME_ASSERT(191 == CM_SURF_FMT_RGBA8_ASTC_10x5); - COMPILE_TIME_ASSERT(192 == CM_SURF_FMT_RGBA8_ASTC_10x6); - COMPILE_TIME_ASSERT(193 == CM_SURF_FMT_RGBA8_ASTC_10x8); - COMPILE_TIME_ASSERT(194 == CM_SURF_FMT_RGBA8_ASTC_10x10); - COMPILE_TIME_ASSERT(195 == CM_SURF_FMT_RGBA8_ASTC_12x10); - COMPILE_TIME_ASSERT(196 == CM_SURF_FMT_RGBA8_ASTC_12x12); - COMPILE_TIME_ASSERT(197 == CM_SURF_FMT_SRGBA8_ASTC_4x4); - COMPILE_TIME_ASSERT(198 == CM_SURF_FMT_SRGBA8_ASTC_5x4); - COMPILE_TIME_ASSERT(199 == CM_SURF_FMT_SRGBA8_ASTC_5x5); - COMPILE_TIME_ASSERT(200 == CM_SURF_FMT_SRGBA8_ASTC_6x5); - COMPILE_TIME_ASSERT(201 == CM_SURF_FMT_SRGBA8_ASTC_6x6); - COMPILE_TIME_ASSERT(202 == CM_SURF_FMT_SRGBA8_ASTC_8x5); - COMPILE_TIME_ASSERT(203 == CM_SURF_FMT_SRGBA8_ASTC_8x6); - COMPILE_TIME_ASSERT(204 == CM_SURF_FMT_SRGBA8_ASTC_8x8); - COMPILE_TIME_ASSERT(205 == CM_SURF_FMT_SRGBA8_ASTC_10x5); - COMPILE_TIME_ASSERT(206 == CM_SURF_FMT_SRGBA8_ASTC_10x6); - COMPILE_TIME_ASSERT(207 == CM_SURF_FMT_SRGBA8_ASTC_10x8); - COMPILE_TIME_ASSERT(208 == CM_SURF_FMT_SRGBA8_ASTC_10x10); - COMPILE_TIME_ASSERT(209 == CM_SURF_FMT_SRGBA8_ASTC_12x10); - COMPILE_TIME_ASSERT(210 == CM_SURF_FMT_SRGBA8_ASTC_12x12); - COMPILE_TIME_ASSERT(211 == CM_SURF_FMT_BGR10_A2UI); - COMPILE_TIME_ASSERT(212 == CM_SURF_FMT_A2_BGR10UI); - COMPILE_TIME_ASSERT(213 == CM_SURF_FMT_A2_RGB10UI); - COMPILE_TIME_ASSERT(214 == CM_SURF_FMT_B5_G6_R5UI); - COMPILE_TIME_ASSERT(215 == CM_SURF_FMT_R5_G6_B5UI); - COMPILE_TIME_ASSERT(216 == CM_SURF_FMT_DEPTH32F_X24_STEN8_UNCLAMPED); - COMPILE_TIME_ASSERT(217 == CM_SURF_FMT_DEPTH32F_UNCLAMPED); - COMPILE_TIME_ASSERT(218 == CM_SURF_FMT_L8_X16_A8_SRGB); - COMPILE_TIME_ASSERT(219 == CM_SURF_FMT_L8_X24_SRGB); - COMPILE_TIME_ASSERT(220 == CM_SURF_FMT_STENCIL8); - COMPILE_TIME_ASSERT(221 == CM_SURF_FMT_L8_SRGB); - COMPILE_TIME_ASSERT(222 == CM_SURF_FMT_R8_SRGB); - COMPILE_TIME_ASSERT(223 == CM_SURF_FMT_RG8_SRGB); -#endif - COMPILE_TIME_ASSERT(cmSurfFmt_LAST == CM_SURF_FMT_RG8_SRGB); - COMPILE_TIME_ASSERT(cmSurfFmt_LAST < 501); -} - -#ifdef ATI_OS_LINUX -typedef void* (*PFNGlxGetProcAddress)(const GLubyte* procName); -static PFNGlxGetProcAddress pfnGlxGetProcAddress=NULL; -static PFNGLXBEGINCLINTEROPAMD glXBeginCLInteropAMD = NULL; -static PFNGLXENDCLINTEROPAMD glXEndCLInteropAMD = NULL; -static PFNGLXRESOURCEATTACHAMD glXResourceAttachAMD = NULL; -static PFNGLXRESOURCEDETACHAMD glxResourceAcquireAMD = NULL; -static PFNGLXRESOURCEDETACHAMD glxResourceReleaseAMD = NULL; -static PFNGLXRESOURCEDETACHAMD glXResourceDetachAMD = NULL; -static PFNGLXGETCONTEXTMVPUINFOAMD glXGetContextMVPUInfoAMD = NULL; -#else -static PFNWGLBEGINCLINTEROPAMD wglBeginCLInteropAMD = NULL; -static PFNWGLENDCLINTEROPAMD wglEndCLInteropAMD = NULL; -static PFNWGLRESOURCEATTACHAMD wglResourceAttachAMD = NULL; -static PFNWGLRESOURCEDETACHAMD wglResourceAcquireAMD = NULL; -static PFNWGLRESOURCEDETACHAMD wglResourceReleaseAMD = NULL; -static PFNWGLRESOURCEDETACHAMD wglResourceDetachAMD = NULL; -static PFNWGLGETCONTEXTGPUINFOAMD wglGetContextGPUInfoAMD = NULL; -#endif - -bool -CALGSLDevice::initGLInteropPrivateExt(CALvoid* GLplatformContext, CALvoid* GLdeviceContext) const -{ -#ifdef ATI_OS_LINUX - GLXContext ctx = (GLXContext)GLplatformContext; - void * pModule = dlopen("libGL.so.1",RTLD_NOW); - - if(NULL == pModule){ - return false; - } - pfnGlxGetProcAddress = (PFNGlxGetProcAddress) dlsym(pModule,"glXGetProcAddress"); - - if (NULL == pfnGlxGetProcAddress){ - return false; - } - - if (!glXBeginCLInteropAMD || !glXEndCLInteropAMD || !glXResourceAttachAMD || - !glXResourceDetachAMD || !glXGetContextMVPUInfoAMD) - { - glXBeginCLInteropAMD = (PFNGLXBEGINCLINTEROPAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXBeginCLInteroperabilityAMD"); - glXEndCLInteropAMD = (PFNGLXENDCLINTEROPAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXEndCLInteroperabilityAMD"); - glXResourceAttachAMD = (PFNGLXRESOURCEATTACHAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXResourceAttachAMD"); - glxResourceAcquireAMD = (PFNGLXRESOURCEDETACHAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXResourceAcquireAMD"); - glxResourceReleaseAMD = (PFNGLXRESOURCEDETACHAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXResourceReleaseAMD"); - glXResourceDetachAMD = (PFNGLXRESOURCEDETACHAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXResourceDetachAMD"); - glXGetContextMVPUInfoAMD = (PFNGLXGETCONTEXTMVPUINFOAMD) pfnGlxGetProcAddress ((const GLubyte *)"glXGetContextMVPUInfoAMD"); - } - - if (!glXBeginCLInteropAMD || !glXEndCLInteropAMD || !glXResourceAttachAMD || - !glXResourceDetachAMD -#ifndef BRAHMA - || !glXGetContextMVPUInfoAMD -#endif - ) - { - return false; - } -#else - if (!wglBeginCLInteropAMD || !wglEndCLInteropAMD || !wglResourceAttachAMD || - !wglResourceDetachAMD || !wglGetContextGPUInfoAMD) - { - HGLRC fakeRC = NULL; - - if (!wglGetCurrentContext()) - { - fakeRC = wglCreateContext((HDC)GLdeviceContext); - wglMakeCurrent((HDC)GLdeviceContext, fakeRC); - } - - wglBeginCLInteropAMD = (PFNWGLBEGINCLINTEROPAMD) wglGetProcAddress ("wglBeginCLInteroperabilityAMD"); - wglEndCLInteropAMD = (PFNWGLENDCLINTEROPAMD) wglGetProcAddress ("wglEndCLInteroperabilityAMD"); - wglResourceAttachAMD = (PFNWGLRESOURCEATTACHAMD) wglGetProcAddress ("wglResourceAttachAMD"); - wglResourceAcquireAMD = (PFNWGLRESOURCEDETACHAMD) wglGetProcAddress ("wglResourceAcquireAMD"); - wglResourceReleaseAMD = (PFNWGLRESOURCEDETACHAMD) wglGetProcAddress ("wglResourceReleaseAMD"); - wglResourceDetachAMD = (PFNWGLRESOURCEDETACHAMD) wglGetProcAddress ("wglResourceDetachAMD"); - wglGetContextGPUInfoAMD = (PFNWGLGETCONTEXTGPUINFOAMD) wglGetProcAddress ("wglGetContextGPUInfoAMD"); - - if (fakeRC) - { - wglMakeCurrent(NULL, NULL); - wglDeleteContext(fakeRC); - } - } - if (!wglBeginCLInteropAMD || !wglEndCLInteropAMD || !wglResourceAttachAMD || - !wglResourceDetachAMD || !wglGetContextGPUInfoAMD) - { - return false; - } -#endif - return true; -} - -bool -CALGSLDevice::glCanInterop(CALvoid* GLplatformContext, CALvoid* GLdeviceContext) -{ - bool canInteroperate = false; - -#ifdef ATI_OS_WIN - LUID glAdapterLuid = {0, 0}; - UINT glChainBitMask = 0; - - LUID calAdapterLuid = {0, 0}; - UINT calChainBitMask = 0; - - HGLRC hRC = (HGLRC)GLplatformContext; - - //get GL context's LUID and chainBitMask from UGL - if (wglGetContextGPUInfoAMD(hRC, &glAdapterLuid, &glChainBitMask)) - { - //now check against the CAL device' LUID and chainBitMask. - if (m_adp->getMVPUinfo(&calAdapterLuid, &calChainBitMask)) - { - canInteroperate = ((glAdapterLuid.HighPart == calAdapterLuid.HighPart) && - (glAdapterLuid.LowPart == calAdapterLuid.LowPart) && - (glChainBitMask == calChainBitMask)); - } - } -#elif defined (ATI_OS_LINUX) -#ifdef BRAHMA - canInteroperate = true; -#else - GLuint glDeviceId = 0 ; - GLuint glChainMask = 0 ; - GLXContext ctx = (GLXContext)GLplatformContext; - - if (glXGetContextMVPUInfoAMD(ctx,&glDeviceId,&glChainMask)){ - GLuint deviceId = 0 ; - GLuint chainMask = 0 ; - - if (m_adp->getMVPUinfo(&deviceId, &chainMask)) - { - // we allow intoperability only with GL context - // reside on a single GPU - if (deviceId == glDeviceId && chainMask == glChainMask){ - canInteroperate = true; - } - } - } -#endif -#endif - return canInteroperate; -} - -bool -CALGSLDevice::glAssociate(CALvoid* GLplatformContext, CALvoid* GLdeviceContext) -{ - //initialize pointers to the gl extension that supports interoperability - if (!initGLInteropPrivateExt(GLplatformContext, GLdeviceContext) || - !glCanInterop(GLplatformContext, GLdeviceContext)) - { - return false; - } - - int flags = 0; - - if (m_adp->pAsicInfo->svmFineGrainSystem) - { - flags = GL_INTEROP_SVM; - } - -#ifdef ATI_OS_LINUX - GLXContext ctx = (GLXContext)GLplatformContext; - return (glXBeginCLInteropAMD(ctx, 0)) ? true : false; -#else - HGLRC hRC = (HGLRC)GLplatformContext; - return (wglBeginCLInteropAMD(hRC, flags)) ? true : false; -#endif -} - -bool -CALGSLDevice::glDissociate(CALvoid* GLplatformContext, CALvoid* GLdeviceContext) -{ - int flags = 0; - - if (m_adp->pAsicInfo->svmFineGrainSystem) - { - flags = GL_INTEROP_SVM; - } - -#ifdef ATI_OS_LINUX - GLXContext ctx = (GLXContext)GLplatformContext; - return (glXEndCLInteropAMD(ctx, 0)) ? true : false; -#else - HGLRC hRC = (HGLRC)GLplatformContext; - return (wglEndCLInteropAMD(hRC, flags)) ? true : false; -#endif -} - -bool -CALGSLDevice::resGLAssociate(GLResAssociate & resData) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - GLResource hRes = {0}; - bool status = false; - cmSurfFmt cal_cmFormat; - uint32 depth; - - gslMemObjectAttribs attribs( - GSL_MOA_TEXTURE_2D, // type - GSL_MOA_MEMORY_ALIAS, // location - GSL_MOA_TILING_TILED, // tiling - GSL_MOA_DISPLAYABLE_NO, // displayable - ATIGL_FALSE, // mipmap - 1, // samples - 0, // cpu_address - GSL_MOA_SIGNED_NO, // signed_format - GSL_MOA_FORMAT_DERIVED, // numFormat - DRIVER_MODULE_GLL, // module - GSL_ALLOCATION_INSTANCED // alloc_type - ); - - hRes.type = resData.type; - - GLResourceData* hData = new GLResourceData; - if (NULL == hData) - { - return false; - } - memset(hData, 0, sizeof(GLResourceData)); - - hRes.name = resData.name; - hRes.flags = resData.flags; - hData->version = GL_RESOURCE_DATA_VERSION; - -#ifdef ATI_OS_LINUX - GLXContext ctx = (GLXContext)resData.GLContext; - if (glXResourceAttachAMD(ctx, &hRes, hData)) - { - attribs.dynamicSharedBufferID = hData->sharedBufferID; - status = true; - } -#else - HGLRC hRC = (HGLRC)resData.GLContext; - if (wglResourceAttachAMD(hRC, &hRes, hData)) - { - status = true; - } -#endif - - if (!status) - { - return false; - } - - // for now, to be safe, allow only textures to have a depth other than 1 - if (hRes.type == GL_RESOURCE_ATTACH_TEXTURE_AMD) - { - depth = hData->rawDimensions.depth; - } - else - { - depth = 1; - } - - attribs.type = static_cast(hData->objectAttribType); - - osAssert(depth <= GLRDATA_MAX_LAYERS); - osAssert(depth >= 1); - attribs.alias_swizzles = (uint32*)malloc(depth * 2 * sizeof(uint32)); - osAssert(attribs.alias_swizzles); - memcpy (attribs.alias_swizzles, hData->swizzles, sizeof(uint32) * depth); - if (hData->levels > 1) - { - attribs.mipmap = ATIGL_TRUE; - attribs.levels = static_cast(hData->levels); - memcpy (&attribs.alias_swizzles[depth], hData->swizzlesMip, sizeof(uint32) * depth); - } - - attribs.cpu_address = (void*)hData->handle; - attribs.alias_subtile = hData->tilingMode; - attribs.mcaddress = hData->cardAddr; - if (hData->isDoppDesktopTexture == GL_TRUE || (hData->isDoppPresentTexture == GL_TRUE) || - (hData->isDisplayable == GL_TRUE)) - { - attribs.isDOPPDesktopTexture = (hData->isDoppDesktopTexture == GL_TRUE); - attribs.displayable = hData->isTilingRotated - ? GSL_MOA_DISPLAYABLE_PORTRAIT_ROTATED : GSL_MOA_DISPLAYABLE_LAYOUT; - } - attribs.doppVidpnSourceId = hData->vidpnSourceId; - - // VBOs are hardcoded to have a UINT8 type format - if (hRes.type == GL_RESOURCE_ATTACH_VERTEXBUFFER_AMD) - { - hData->format = CM_SURF_FMT_LUMINANCE8; - } - // CAL supports only a limited number of cm_surf formats, so we - // have to translate incoming cm_surf formats - uint32 index = hData->format - (uint32)CM_SURF_FMT_LUMINANCE8; - if (index >= sizeof(cmFormatXlateTable)/sizeof(cmFormatXlateParams)) - { - free(attribs.alias_swizzles); - delete hData; - return false; - } - osAssert(static_cast(hData->format) == cmFormatXlateTable[index].raw_cmFormat); - cal_cmFormat = cmFormatXlateTable[index].cal_cmFormat; - if (cal_cmFormat == 500) - { - free(attribs.alias_swizzles); - delete hData; - return false; // format is not supported by CAL - } - attribs.channelOrder = cmFormatXlateTable[index].channelOrder; - attribs.alias_perSurfTileInfo = hData->perSurfTileInfo; - attribs.alias_GLInterop = ATIGL_TRUE; - attribs.numFormat = GSL_MOA_FORMAT_DERIVED; - - gslMemObject mem; - - if (hData->offset != 0) - { - osAssert((hData->rawDimensions.height == 1) && (depth == 1)); - mem = m_cs->createMemObject2D(CM_SURF_FMT_LUMINANCE8, hData->surfaceSize, 1, &attribs); - } - else - { - mem = m_cs->createMemObject3D(cal_cmFormat, hData->paddedDimensions.width, - hData->rawDimensions.height, depth, &attribs); - } - if (hRes.type == GL_RESOURCE_ATTACH_VERTEXBUFFER_AMD) - { - attribs.tiling = mem->getAttribs().tiling; - resData.mem_base = mem; - mem = m_cs->createOffsetMemObject2D(resData.mem_base, (static_cast(hData->offset)), - cal_cmFormat, - hData->paddedDimensions.width, - 1, &attribs); - } - else if ((hData->offset != 0) && (hData->rawDimensions.height == 1) && (depth == 1)) - { - resData.mem_base = mem; - attribs.tiling = mem->getAttribs().tiling; - mem = m_cs->createOffsetMemObject3D(resData.mem_base, (static_cast(hData->offset)), - cal_cmFormat, hData->paddedDimensions.width, - hData->rawDimensions.height, depth, &attribs); - } - free (attribs.alias_swizzles); - resData.mbResHandle = (CALvoid*)hData->mbResHandle; - resData.memObject = mem; - delete hData; - return mem != 0; -} - -bool -CALGSLDevice::resGLAcquire(CALvoid* GLplatformContext, - CALvoid* mbResHandle, - CALuint type) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - GLResource hRes = {0}; - osAssert(mbResHandle); - hRes.mbResHandle = (GLuintp)mbResHandle; - hRes.type = type; - -#ifdef ATI_OS_LINUX - GLXContext ctx = (GLXContext) GLplatformContext; - return (glxResourceAcquireAMD(ctx, &hRes)) ? true : false; -#else - HGLRC hRC = wglGetCurrentContext(); - //! @todo A temporary workaround for MT issue in conformance fence_sync - if (0 == hRC) { - return true; - } - return (wglResourceAcquireAMD(hRC, &hRes)) ? true : false; -#endif -} - -bool -CALGSLDevice::resGLRelease(CALvoid* GLplatformContext, - CALvoid* mbResHandle, - CALuint type) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - GLResource hRes = {0}; - osAssert(mbResHandle); - hRes.mbResHandle = (GLuintp)mbResHandle; - hRes.type = type; - -#ifdef ATI_OS_LINUX - //TODO : make sure the application GL context is current. if not no - // point calling into the GL RT. - GLXContext ctx = (GLXContext) GLplatformContext; - return (glxResourceReleaseAMD(ctx, &hRes)) ? true : false; -#else - // Make the call into the GL driver only if the application GL context is current - HGLRC hRC = wglGetCurrentContext(); - //! @todo A temporary workaround for MT issue in conformance fence_sync - if (0 == hRC) { - return true; - } - return (wglResourceReleaseAMD(hRC, &hRes)) ? true : false; -#endif -} - -bool -CALGSLDevice::resGLFree ( - CALvoid* GLplatformContext, - CALvoid* GLdeviceContext, - gslMemObject mem, - gslMemObject mem_base, - CALvoid* mbResHandle, - CALuint type) const -{ - //! @note: GSL device isn't thread safe - amd::ScopedLock k(gslDeviceOps()); - - GLResource hRes = {0}; - - osAssert(mbResHandle); - hRes.mbResHandle = (GLuintp)mbResHandle; - hRes.type = type; - - if (mem_base) - { - m_cs->destroyMemObject(mem_base); - } - m_cs->destroyMemObject(mem); - -#ifdef ATI_OS_LINUX - GLXContext ctx = (GLXContext)GLplatformContext; - return (glXResourceDetachAMD(ctx, &hRes)) ? true : false; -#else - HGLRC hRC = (HGLRC)GLplatformContext; - return (wglResourceDetachAMD(hRC, &hRes)) ? true : false; -#endif -}; diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLStubs.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLStubs.cpp deleted file mode 100644 index 731023d1d5..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/GSLStubs.cpp +++ /dev/null @@ -1,154 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "os_if.h" -#include "osws_if.h" - -#include "atidefines.h" -#include "atitypes.h" -#include "scl_types.h" -#include "SCInterface.h" - -// -// This file represents the entry points that are stubbed out to satisfy the -// linker, but aren't used in the runtime of GSL operations. -// - - -enum fsComponentType { - FS_BYTE, - FS_UNSIGNED_BYTE, - FS_SHORT, - FS_UNSIGNED_SHORT, - FS_INT, - FS_UNSIGNED_INT, - FS_FLOAT, - FS_FLOAT16, -}; - -enum fsInstrSet { - FS_INSTR_KHAN, ///< Generate Khan based instruction set - FS_INSTR_PELE ///< Generate Pele based instruction set -}; - -enum fsUsage { - FS_USAGE_HW, ///< An actual hardware stream - FS_USAGE_SW ///< A place holder stream (to support SW path) -}; - -struct fsInstr { - fsUsage usage; ///< How the stream is going to be used (place holder or actual hardware stream) - uint32 components; ///< Number of components to the input vector - fsComponentType type; ///< Type of each component - bool32 normalize; ///< Should the components be normalized to the -1..1 range - uint32 stride; ///< Stride between vectors - uint32 ivmOffset; ///< location in input vector memory -}; - - -sclHandle CONV -sclInit(const sclShaderConstantAddress* shaderStateConstTable, - const sclProfile& profile, - const sclLimits& fpLimits, - const sclLimits& vpLimits) -{ - return 0; -} - -void CONV -sclDestroy(sclHandle hSCL) -{ -} - -sclProgram* CONV -sclCompile(sclHandle hSCL, - const sclInputShader& shader, - const sclCompilerParams& params, - const sclLimits& limits) -{ - return 0; -} - -sclProgramPair* CONV -sclLink(sclHandle hSCL, - const sclInputMultShaderPair *shader, - const sclCompilerParams& params, - const sclLimits& fpLimits, - const sclLimits& vpLimits) -{ - return 0; -} - -void CONV -sclFreeProgram(sclHandle hSCL, - sclProgram* program) -{ -} - -sclShaderReplaceHandle CONV -sclRegisterShaderString(sclHandle hSCL, - const sclInputShader& src, - const sclInputShader& dst) -{ - return 0; -} - -void CONV -sclUnregisterShaderString(sclHandle hSCL, - sclShaderReplaceHandle hReplacement) -{ -} - -bool32 CONV -fsCompile(fsInstrSet instrSet, - uint32 instrCount, - const fsInstr* instr, - void*& binary, - uint32& length, - bool32 dumpShader, - bool32 doCacheOpt, - const sclCompilerParamTessellation& tessParams) -{ - return ATIGL_TRUE; -} - - -void CONV -fsFreeBinary(void* binary) -{ -} - - -void CONV -oswsInit(HOSInstance hOSInst) -{ - // - // do nothing... - // -} - -void CONV -oswsExit() -{ - // - // do nothing... - // -} - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.cpp deleted file mode 100644 index 6aea71353d..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.cpp +++ /dev/null @@ -1,194 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "gsl_ctx.h" -#include "GSLContext.h" -#include "backend.h" -#include "GSLDevice.h" -#include "amuABI.h" - -bool -getFuncInfoFromImage(CALimage image, CALfuncInfo *pFuncInfo) -{ - if (image == 0) - { - return false; - } - - if (pFuncInfo == 0) - { - return false; - } - - //Initialize the pFuncInfo - pFuncInfo->maxScratchRegsNeeded = 0; - pFuncInfo->numSharedGPRUser = 0; - pFuncInfo->numSharedGPRTotal = 0; - pFuncInfo->numThreadPerGroup = 0; - pFuncInfo->numThreadPerGroupX = 0; - pFuncInfo->numThreadPerGroupY = 0; - pFuncInfo->numThreadPerGroupZ = 0; - pFuncInfo->totalNumThreadGroup = 0; - pFuncInfo->numWavefrontPerSIMD = 0; - pFuncInfo->setBufferForNumGroup = false; - pFuncInfo->wavefrontSize = 0; - pFuncInfo->numGPRsAvailable = 0; - pFuncInfo->numGPRsUsed = 0; - pFuncInfo->numSGPRsAvailable = 0; - pFuncInfo->numSGPRsUsed = 0; - pFuncInfo->numVGPRsAvailable = 0; - pFuncInfo->numVGPRsUsed = 0; - pFuncInfo->LDSSizeAvailable = 0; - pFuncInfo->LDSSizeUsed = 0; - pFuncInfo->stackSizeAvailable = 0; - pFuncInfo->stackSizeUsed = 0; - - //read data from image file - AMUabiMultiBinary mb; - amuABIMultiBinaryCreate(&mb); - if (!amuABIMultiBinaryUnpack(mb, (void*) image)) - { - amuABIMultiBinaryDestroy(mb); - return false; - } - - unsigned int encodingCount; - if (!amuABIMultiBinaryGetEncodingCount(&encodingCount, mb)) - { - amuABIMultiBinaryDestroy(mb); - return false; - } - - AMUabiEncoding encoding; - //get encoding info for the first encoding - if ((encodingCount > 0)&& !amuABIMultiBinaryGetEncoding( &encoding, mb, 0)) - { - amuABIMultiBinaryDestroy(mb); - return false; - } - - unsigned int machine, type; - if (!amuABIEncodingGetSignature(&machine, &type, encoding)) - { - amuABIMultiBinaryDestroy(mb); - return false; - } - - if (!amuABIMultiBinaryFindEncoding(&encoding, mb, machine, type)) - { - amuABIMultiBinaryDestroy(mb); - return false; - } - - unsigned int progInfosCount = 0; - CALProgramInfoEntry* pInfos = 0; - if (!amuABIEncodingGetProgInfos(&progInfosCount, &pInfos, encoding)) - { - amuABIMultiBinaryDestroy(mb); - return false; - } - - for (CALuint i =0; i < progInfosCount; i++) - { - switch(pInfos[i].address) - { - case AMU_ABI_CS_MAX_SCRATCH_REGS: - pFuncInfo->maxScratchRegsNeeded = pInfos[i].value; - break; - case AMU_ABI_CS_NUM_SHARED_GPR_USER: - pFuncInfo->numSharedGPRUser = pInfos[i].value; - break; - case AMU_ABI_CS_NUM_SHARED_GPR_TOTAL: - pFuncInfo->numSharedGPRTotal = pInfos[i].value; - break; - case AMU_ABI_ECS_SETUP_MODE: - break; - case AMU_ABI_NUM_THREAD_PER_GROUP: - pFuncInfo->numThreadPerGroup = pInfos[i].value; - break; - case AMU_ABI_NUM_THREAD_PER_GROUP_X: - pFuncInfo->numThreadPerGroupX = pInfos[i].value; - break; - case AMU_ABI_NUM_THREAD_PER_GROUP_Y: - pFuncInfo->numThreadPerGroupY = pInfos[i].value; - break; - case AMU_ABI_NUM_THREAD_PER_GROUP_Z: - pFuncInfo->numThreadPerGroupZ = pInfos[i].value; - break; - case AMU_ABI_TOTAL_NUM_THREAD_GROUP: - pFuncInfo->totalNumThreadGroup = pInfos[i].value; - break; - case AMU_ABI_NUM_WAVEFRONT_PER_SIMD: - case AMU_ABI_MAX_WAVEFRONT_PER_SIMD: //CAL_USE_SC_PRM - pFuncInfo->numWavefrontPerSIMD = pInfos[i].value; - break; - case AMU_ABI_IS_MAX_NUM_WAVE_PER_SIMD: - break; - case AMU_ABI_SET_BUFFER_FOR_NUM_GROUP: - pFuncInfo->setBufferForNumGroup = (0 != pInfos[i].value) ? true : false; - break; - case AMU_ABI_WAVEFRONT_SIZE: - pFuncInfo->wavefrontSize = pInfos[i].value; - break; - case AMU_ABI_NUM_GPR_AVAIL: - pFuncInfo->numGPRsAvailable = pInfos[i].value; - break; - case AMU_ABI_NUM_GPR_USED: - pFuncInfo->numGPRsUsed = pInfos[i].value; - break; - case AMU_ABI_LDS_SIZE_AVAIL: - pFuncInfo->LDSSizeAvailable = pInfos[i].value; - break; - case AMU_ABI_LDS_SIZE_USED: - pFuncInfo->LDSSizeUsed = pInfos[i].value; - break; - case AMU_ABI_STACK_SIZE_AVAIL: - pFuncInfo->stackSizeAvailable = pInfos[i].value; - break; - case AMU_ABI_STACK_SIZE_USED: - pFuncInfo->stackSizeUsed = pInfos[i].value; - break; - case AMU_ABI_SI_NUM_SGPRS_AVAIL: - pFuncInfo->numSGPRsAvailable = pInfos[i].value; - break; - case AMU_ABI_SI_NUM_SGPRS: - pFuncInfo->numSGPRsUsed = pInfos[i].value; - break; - case AMU_ABI_SI_NUM_VGPRS_AVAIL: - pFuncInfo->numVGPRsAvailable = pInfos[i].value; - break; - case AMU_ABI_SI_NUM_VGPRS: - pFuncInfo->numVGPRsUsed = pInfos[i].value; - break; - - default: - //GSLAssert(0 && "Unknown address in program info"); - break; - } - } - - amuABIEncodingGetScratchRegisterCount(&pFuncInfo->maxScratchRegsNeeded, encoding); - - amuABIMultiBinaryDestroy(mb); - - return true; -} - -gslMemObjectAttribTiling g_CALBETiling_Tiled = GSL_MOA_TILING_TILED; diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.h deleted file mode 100644 index 5970aff64b..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/backend.h +++ /dev/null @@ -1,225 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __BACKEND_H__ -#define __BACKEND_H__ - -#include -#include - -//internal -#include "gsl_enum.h" -#include "gsl_types.h" -#include "cm_enum.h" -#include "caltarget.h" - -typedef void CALvoid; /**< void type */ -typedef char CALchar; /**< ASCII character */ -typedef signed char CALbyte; /**< 1 byte signed integer value */ -typedef unsigned char CALubyte; /**< 1 byte unsigned integer value */ -typedef signed short CALshort; /**< 2 byte signed integer value */ -typedef unsigned short CALushort; /**< 2 byte unsigned integer value */ -typedef signed int CALint; /**< 4 byte signed integer value */ -typedef unsigned int CALuint; /**< 4 byte unsigned intger value */ -typedef float CALfloat; /**< 32-bit IEEE floating point value */ -typedef double CALdouble; /**< 64-bit IEEE floating point value */ -typedef signed long CALlong; /**< long value */ -typedef unsigned long CALulong; /**< unsigned long value */ - -#if defined(_MSC_VER) - -typedef signed __int64 CALint64; /**< 8 byte signed integer value */ -typedef unsigned __int64 CALuint64; /**< 8 byte unsigned integer value */ - -#elif defined(__GNUC__) - -typedef signed long long CALint64; /**< 8 byte signed integer value */ -typedef unsigned long long CALuint64; /**< 8 byte unsigned integer value */ - -#else -#error "Unsupported compiler type." -#endif - -/** Boolean type */ -typedef enum CALbooleanEnum { - CAL_FALSE = 0, /**< Boolean false value */ - CAL_TRUE = 1 /**< Boolean true value */ -} CALboolean; - -/** CAL image container */ -typedef struct CALimageRec* CALimage; - -#define CAL_ASIC_INFO_MAX_LEN 128 -#define CAL_DRIVER_STORE_MAX_LEN 200 - -/** CAL device attributes */ -typedef struct CALdeviceattribsRec { - CALuint struct_size; /**< Client filled out size of CALdeviceattribs struct */ - CALtarget target; /**< Asic identifier */ - CALuint localRAM; /**< Amount of local GPU RAM in megabytes */ - CALuint uncachedRemoteRAM; /**< Amount of uncached remote GPU memory in megabytes */ - CALuint cachedRemoteRAM; /**< Amount of cached remote GPU memory in megabytes */ - CALuint engineClock; /**< GPU device clock rate in megahertz */ - CALuint memoryClock; /**< GPU memory clock rate in megahertz */ - CALuint wavefrontSize; /**< Wavefront size */ - CALuint numberOfSIMD; /**< Number of SIMDs */ - CALuint numberOfCUsperShaderArray; /**< Number of CUs per shader array */ - bool doublePrecision; /**< double precision supported */ - CALuint numberOfShaderEngines; /**< Number of shader engines */ - CALuint totalVisibleHeap; /**< Amount of visible local GPU RAM in megabytes */ - CALuint totalInvisibleHeap; /**< Amount of invisible local GPU RAM in megabytes */ - CALuint totalDirectHeap; /**< Amount of direct GPU memory in megabytes */ - CALuint totalCoherentHeap; /**< Amount of coherent GPU memory in megabytes */ - CALuint totalRemoteSharedHeap; /**< Amount of remote Shared GPU memory in megabytes */ - CALuint totalCachedRemoteSharedHeap; /**< Amount of cached remote Shared GPU memory in megabytes */ - CALuint totalSDIHeap; /**< Amount of SDI memory allocated in CCC */ - CALuint pciTopologyInformation; /**< PCI topology information contains: bus, device and function number. */ - CALchar boardName[CAL_ASIC_INFO_MAX_LEN]; /**< Actual ASIC board name and not the internal name. */ - CALuint memBusWidth; /**< Memory busw width */ - CALuint numMemBanks; /**< Number of memory banks */ - CALuint counterFreq; /**< Ref clock counter frequency */ - double nanoSecondsPerTick; /**< Nano seconds per GPU tick */ - bool longIdleDetect; /**< Whether LongIdleDetect enabled */ - bool svmAtomics; /**< check if svm atomics support */ - CALuint64 vaStart; /**< VA start address */ - CALuint64 vaEnd; /**< VA end address */ - bool isWorkstation; /**< Whether Device is a Workstation/Server part */ - CALuint numOfVpu; /**< number of vpu in the device*/ - bool isOpenCL200Device; /**< the flag to mark if the device is OpenCL 200 */ - bool isSVMFineGrainSystem; /**< check if SVM finegrainsystem */ - bool isWDDM2Enabled; /**< check if WDDM2 is enabled */ - CALuint maxRTCUs; /**< The maximum number of RT CUs for RT queues */ - CALuint asicRevision; /**< The ASIC revision ID */ - CALchar driverStore[CAL_DRIVER_STORE_MAX_LEN];/**< Driver store location. */ - CALuint pcieDeviceID; /**< The ASIC PCIE device ID */ - CALuint pcieRevisionID; /**< The ASIC PCIE revision ID */ -} CALdeviceattribs; - - -/** CAL resource allocation flags **/ -typedef enum CALresallocflagsEnum { - CAL_RESALLOC_GLOBAL_BUFFER = 1, /**< used for global import/export buffer */ -} CALresallocflags; - - -/** CAL function information **/ -typedef struct CALfuncInfoRec -{ - CALuint maxScratchRegsNeeded; /**< Maximum number of scratch regs needed */ - CALuint numSharedGPRUser; /**< Number of shared GPRs */ - CALuint numSharedGPRTotal; /**< Number of shared GPRs including ones used by SC */ - CALuint numThreadPerGroup; /**< Flattend umber of threads per group */ - CALuint numThreadPerGroupX; /**< x dimension of numThreadPerGroup */ - CALuint numThreadPerGroupY; /**< y dimension of numThreadPerGroup */ - CALuint numThreadPerGroupZ; /**< z dimension of numThreadPerGroup */ - CALuint totalNumThreadGroup; /**< Total number of thread groups */ - CALuint numWavefrontPerSIMD; /**< Number of wavefronts per SIMD */ - bool setBufferForNumGroup; /**< Need to set up buffer for info on number of thread groups? */ - CALuint wavefrontSize; /**< number of threads per wavefront. */ - CALuint numGPRsAvailable; /**< number of GPRs available to the program */ - CALuint numGPRsUsed; /**< number of GPRs used by the program */ - CALuint LDSSizeAvailable; /**< LDS size available to the program */ - CALuint LDSSizeUsed; /**< LDS size used by the program */ - CALuint stackSizeAvailable; /**< stack size availabe to the program */ - CALuint stackSizeUsed; /**< stack size use by the program */ - CALuint numSGPRsAvailable; /**< number of SGPRs available to the program */ - CALuint numSGPRsUsed; /**< number of SGPRs used by the program */ - CALuint numVGPRsAvailable; /**< number of VGPRs available to the program */ - CALuint numVGPRsUsed; /**< number of VGPRs used by the program */ -} CALfuncInfo; - -// flags for calCtxWaitForEvents -typedef enum CALwaitTypeEnum -{ - CAL_WAIT_POLLING = 0, - CAL_WAIT_LOW_CPU_UTILIZATION = 1, -} CALwaitType; - -// -// calResAllocView typedefs -// -typedef enum CALresallocviewflagsRec { - CAL_RESALLOCVIEW_GLOBAL_BUFFER = CAL_RESALLOC_GLOBAL_BUFFER, /**< used for global import/export buffer */ - CAL_RESALLOCVIEW_LINEAR_ALIGNED = CAL_RESALLOC_GLOBAL_BUFFER, /**< 256 byte alignment restriction. */ - CAL_RESALLOCVIEW_LINEAR_UNALIGNED = 3, /**< no alignment restrictions */ -} CALresallocviewflags; - -typedef struct CALresourceDescRec { - gslMemObjectAttribLocation type; - gslResource3D size; - cmSurfFmt format; - gslChannelOrder channelOrder; - gslMemObjectAttribType dimension; - CALuint mipLevels; - CALvoid* systemMemory; - CALuint flags; - CALuint systemMemorySize; - CALuint64 busAddress[2]; - mcaddr vaBase; - gslMemObjectAttribSection section; - CALuint minAlignment; - bool isAllocSVM; - bool isAllocExecute; -} CALresourceDesc; - -typedef enum CALresallocsliceviewflagsRec { - CAL_RESALLOCSLICEVIEW_GLOBAL_BUFFER = CAL_RESALLOC_GLOBAL_BUFFER, /**< used for global import/export buffer */ - CAL_RESALLOCSLICEVIEW_LINEAR_ALIGNED = CAL_RESALLOC_GLOBAL_BUFFER, /**< 256 byte alignment restriction. */ - CAL_RESALLOCSLICEVIEW_LINEAR_UNALIGNED = CAL_RESALLOCVIEW_LINEAR_UNALIGNED, /**< no alignment restrictions */ - CAL_RESALLOCSLICEVIEW_LEVEL = 0x10, /**< sliceDesc.layer is not used, the whole level is only*/ - CAL_RESALLOCSLICEVIEW_LAYER = 0x20, /**< sliceDesc.layer is not used, the whole level is only*/ - CAL_RESALLOCSLICEVIEW_LEVEL_AND_LAYER = CAL_RESALLOCSLICEVIEW_LEVEL | CAL_RESALLOCSLICEVIEW_LAYER, -} CALresallocsliceviewflags; - -typedef enum CALmemcopyflagsEnum -{ - CAL_MEMCOPY_DEFAULT = 0, /**< default CAL behavior of partial sync */ - CAL_MEMCOPY_SYNC = 1, /**< used to synchronize with the specified CAL context */ - CAL_MEMCOPY_ASYNC = 2, /**< used to indicate completely asynchronous behavior */ -} CALmemcopyflags; - -class CALGSLDevice; - -//! Engine types -enum EngineType -{ - MainEngine = 0, - SdmaEngine, - AllEngines -}; - -struct GpuEvent -{ - static constexpr unsigned int InvalidID = ((1<<30) - 1); - - EngineType engineId_; ///< type of the id - unsigned int id; ///< actual event id - - //! GPU event default constructor - GpuEvent(): engineId_(MainEngine), id(InvalidID) {} - - //! Returns true if the current event is valid - bool isValid() const { return (id != InvalidID) ? true : false; } - - //! Set invalid event id - void invalidate() { id = InvalidID; } -}; - -#endif diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/caltarget.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/caltarget.h deleted file mode 100644 index 2f5b0fb6ce..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/caltarget.h +++ /dev/null @@ -1,79 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __CALTARGET_H__ -#define __CALTARGET_H__ - -/** Device Kernel ISA */ -typedef enum CALtargetEnum { - CAL_TARGET_600, /**< R600 GPU ISA */ - CAL_TARGET_610, /**< RV610 GPU ISA */ - CAL_TARGET_630, /**< RV630 GPU ISA */ - CAL_TARGET_670, /**< RV670 GPU ISA */ - CAL_TARGET_7XX, /**< R700 class GPU ISA */ - CAL_TARGET_770, /**< RV770 GPU ISA */ - CAL_TARGET_710, /**< RV710 GPU ISA */ - CAL_TARGET_730, /**< RV730 GPU ISA */ - CAL_TARGET_CYPRESS, /**< CYPRESS GPU ISA */ - CAL_TARGET_JUNIPER, /**< JUNIPER GPU ISA */ - CAL_TARGET_REDWOOD, /**< REDWOOD GPU ISA */ - CAL_TARGET_CEDAR, /**< CEDAR GPU ISA */ -//##BEGIN_PRIVATE## - CAL_TARGET_SUMO, /**< SUMO GPU ISA */ - CAL_TARGET_SUPERSUMO, /**< SUPERSUMO GPU ISA */ - CAL_TARGET_WRESTLER, /**< WRESTLER GPU ISA */ - CAL_TARGET_CAYMAN, /**< CAYMAN GPU ISA */ - CAL_TARGET_KAUAI, /**< KAUAI GPU ISA */ - CAL_TARGET_BARTS , /**< BARTS GPU ISA */ - CAL_TARGET_TURKS , /**< TURKS GPU ISA */ - CAL_TARGET_CAICOS, /**< CAICOS GPU ISA */ - CAL_TARGET_TAHITI, /**< TAHITI GPU ISA*/ - CAL_TARGET_PITCAIRN, /**< PITCAIRN GPU ISA*/ - CAL_TARGET_CAPEVERDE, /**< CAPE VERDE GPU ISA*/ - CAL_TARGET_DEVASTATOR, /**< DEVASTATOR GPU ISA*/ - CAL_TARGET_SCRAPPER, /**< SCRAPPER GPU ISA*/ - CAL_TARGET_OLAND, /**< OLAND GPU ISA*/ - CAL_TARGET_BONAIRE, /**< BONAIRE GPU ISA*/ - CAL_TARGET_SPECTRE, /**< KAVERI1 GPU ISA*/ - CAL_TARGET_SPOOKY, /**< KAVERI2 GPU ISA*/ - CAL_TARGET_KALINDI, /**< KALINDI GPU ISA*/ - CAL_TARGET_HAINAN, /**< HAINAN GPU ISA*/ - CAL_TARGET_HAWAII, /**< HAWAII GPU ISA*/ - CAL_TARGET_ICELAND, /**< ICELAND GPU ISA*/ - CAL_TARGET_TONGA, /**< TONGA GPU ISA*/ - CAL_TARGET_GODAVARI, /**< MULLINS GPU ISA*/ - CAL_TARGET_FIJI, /**< FIJI GPU ISA*/ - CAL_TARGET_CARRIZO, /**< CARRIZO GPU ISA*/ - CAL_TARGET_ELLESMERE, /**< ELLESMERE GPU ISA*/ - CAL_TARGET_BAFFIN, /**< BAFFIN GPU ISA*/ - CAL_TARGET_GREENLAND, /**< GREENLAND GPU ISA*/ - CAL_TARGET_STONEY, /**< STONEY GPU ISA*/ - CAL_TARGET_LEXA, /**< LEXA GPU ISA*/ - CAL_TARGET_RAVEN, /**< RAVEN GPU ISA*/ - CAL_TARGET_RAVEN2, /**< RAVEN2 GPU ISA*/ - CAL_TARGET_RENOIR, /**< RENOIR GPU ISA*/ - CAL_TARGET_POLARIS22, /**< POLARIS22 GPU ISA*/ - CAL_TARGET_VEGA12, /**< VEGA12 GPU ISA*/ - CAL_TARGET_VEGA20, /**< VEGA20 GPU ISA*/ - CAL_TARGET_LAST = CAL_TARGET_VEGA20, /**< last */ -//##END_PRIVATE## -} CALtarget; - -#endif diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.cpp deleted file mode 100644 index 757ffef8cd..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.cpp +++ /dev/null @@ -1,115 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#include "inifile.h" -#include "ini_export.h" -#include "ini_values.h" - -#include "gsl_enum.h" - -extern gslMemObjectAttribTiling g_CALBETiling_Tiled; - -void -getConfigFromFile(gslStaticRuntimeConfig& scfg, - gslDynamicRuntimeConfig& dcfg) -{ - const char* calIniFile = getenv("CAL_INI_FILE"); - IniFile iniFile(cmString(calIniFile ? calIniFile : INI_FILE)); - - CALboolean dumpIL = CAL_FALSE; - CALboolean dumpISA = CAL_FALSE; - CALboolean macro = CAL_TRUE; - CALboolean micro = CAL_TRUE; - CALboolean breakonload = CAL_FALSE; - CALint useRectPrim = 0; - CALboolean forceRemoteMemory = CAL_FALSE; - CALboolean disableAsyncDma = CAL_FALSE; - CALboolean disableVM = CAL_FALSE; - - - dcfg.bEmulator.hasValue = ATIGL_TRUE; - dcfg.DropFlush.hasValue = ATIGL_TRUE; - dcfg.EnableCommandbufferDump.hasValue = ATIGL_TRUE; - dcfg.WaitForIdleAfterSubmit.hasValue = ATIGL_TRUE; - dcfg.FlushAfterRender.hasValue = ATIGL_TRUE; - dcfg.nPatchDumpLevel.hasValue = ATIGL_TRUE; - - cmString commandbufferDumpFilename; - - iniFile.getValue(section, CAL_EMULATOR, (CALboolean*) &dcfg.bEmulator.value); - iniFile.getValue(section, CAL_ENABLE_FORCE_ASIC_ID, (CALboolean*) &dcfg.forceAsicID.hasValue); - iniFile.getValue(section, CAL_FORCE_ASIC_ID, (CALint*) &dcfg.forceAsicID.value); - iniFile.getValue(section, CAL_DROPFLUSH, (CALboolean*) &dcfg.DropFlush.value); - iniFile.getValue(section, CAL_ENABLEPACKETDUMP, (CALboolean*) &dcfg.EnableCommandbufferDump.value); - - // Check if location string is longer than 128 then assign, if not default location will be C:\packet.txt in gsl_ctx.cpp: gsCtxManager::PacketDump() - uintp length = commandbufferDumpFilename.length(); - if (length > 0 && length < sizeof(dcfg.CommandbufferDumpFilename)) - ::strncpy(dcfg.CommandbufferDumpFilename, commandbufferDumpFilename.c_str(), sizeof(dcfg.CommandbufferDumpFilename) - 1); - - iniFile.getValue(section, CAL_ENABLEPATCHDUMP, (CALint*) &dcfg.nPatchDumpLevel.value); - iniFile.getValue(section, CAL_ENABLEMACROTILE, (CALboolean*) ¯o); - iniFile.getValue(section, CAL_ENABLEMICROTILE, (CALboolean*) µ); - iniFile.getValue(section, CAL_BREAK_ON_LOAD, (CALboolean*) &breakonload); - iniFile.getValue(section, CAL_FORCE_REMOTE_MEMORY, (CALboolean*) &forceRemoteMemory); - iniFile.getValue(section, CAL_DISABLE_ASYNC_DMA, (CALboolean*) &disableAsyncDma); - iniFile.getValue(section, CAL_WAITFORIDLEAFTERSUBMIT, (CALboolean*) &dcfg.WaitForIdleAfterSubmit.value); - iniFile.getValue(section, CAL_ENABLE_DUMP_IL, (CALboolean*) &dumpIL); - iniFile.getValue(section, CAL_ENABLE_DUMP_ISA, (CALboolean*) &dumpISA); - iniFile.getValue(section, CAL_ENABLE_FLUSH_AFTER_RENDER, (CALboolean*) &dcfg.FlushAfterRender.value); - iniFile.getValue(section, CAL_DISABLE_VM, (CALboolean*) &disableVM); - - if (disableVM) - { - scfg.VMMode = GSL_CONFIG_VM_MODE_FORCE_OFF; - } - - if (!macro && !micro) - { - g_CALBETiling_Tiled = GSL_MOA_TILING_LINEAR; - } - - if (breakonload) - { - #ifndef ATI_OS_LINUX - __debugbreak(); - #endif - } - - switch (forceRemoteMemory) - { - case 1: - // - // Also set linear, due to CAL expectations about different memory regions - // - g_CALBETiling_Tiled = GSL_MOA_TILING_LINEAR; - break; - - default: - break; - } - - if (disableAsyncDma) - { - dcfg.drmdmaMode.hasValue = ATIGL_TRUE; - dcfg.drmdmaMode.value = GSL_CONFIG_DRMDMA_MODE_FORCE_OFF; - } -} - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.h deleted file mode 100644 index 43f489fce2..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_export.h +++ /dev/null @@ -1,33 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __INI_EXPORT_H__ -#define __INI_EXPORT_H__ - -#include "gsl_config.h" - -void -getConfigFromFile(gslStaticRuntimeConfig& scfg, - gslDynamicRuntimeConfig& dcfg); - -#endif - - - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_values.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_values.h deleted file mode 100644 index 289fb8a6fa..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/ini_values.h +++ /dev/null @@ -1,354 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef __INI_VALUES_H__ -#define __INI_VALUES_H__ - -#include "cm_string.h" - - -const cmString section("CAL"); -const cmString INI_FILE("cal.ini"); - -/* VSYNC COMMENTS -0 - always off -1 - app preference (default off) -2 - app preference (default on) -3 - always on -*/ -const cmString CAL_OGLWAITVERTICALSYNC("VSyncControl"); - -// Private panel setting for V-sync control -const cmString CAL_ENABLETEARFREESWAP("VSyncControl"); - -// Public panel setting to set max anisotropy: 0=app pref, 2=2x, 4=4x, 8=8x, 16=16x -const cmString CAL_OGLMAXANISOTROPY("MaxAnisotropy"); - -// Public panel setting to select performance Aniso -const cmString CAL_OGLANISOPERF("AnisoPerf"); - - -// Public panel setting to select quality mode -const cmString CAL_OGLANISOQUAL("AnisoQuality"); - -// Public panel setting -const cmString CAL_OGLANISOTYPE("AnisoType"); - -// Private panel -const cmString CAL_ENABLEANISOTROPICFILTERING("AnisoFiltering"); - -// Public panel setting -const cmString CAL_OGLALIASSLIDER("AnisoDegree"); - -// Public panel setting for LOD bias; ranges from 0(high quality) to 3(high performance); -const cmString CAL_OGLLODBIAS("TextureLod"); - -// Public panel setting to force Z buffer depth -const cmString CAL_OGLFORCEZBUFFERDEPTH("ForceZBufferDepth"); - -// Public panel setting to select alpha dither method -const cmString CAL_OGLALPHADITHERMETHOD("DitherAlpha"); - -// Private Panel Setting for setting multisample value for FSAA -const cmString CAL_MULTISAMPLE("Multisample"); - -// Public Panel setting for forcing AA -const cmString CAL_ACE_OGLENABLEFSAA("AntiAlias"); - -// Public panel setting to Enable fast full scene anti-aliasing -const cmString CAL_OGLENABLEFASTFULLSCENEAA("FSAAPerfMode"); - -//Private Panel setting to force FSAA on -const cmString CAL_ENABLEFASTFULLSCENEAA("FastFullSceneAntiAlias"); - - -// Public panel setting to set full scene anti-aliasing scale. -// Acceptable values are 0, 2-6 -const cmString CAL_OGLFULLSCENEAASCALE("AntiAliasSamples"); - -// Private panel setting to force FSAA, Acceptable values are 0, 2-6. -const cmString CAL_FULLSCENEAASCALE("FullSceneAntiAliasScale"); - -// Public panel setting to enable triple-buffering -const cmString CAL_OGLENABLETRIPLEBUFFERING("EnableTripleBuffering"); - -// Public panel setting to set texture optimization -const cmString CAL_OGLTEXTUREOPT("TextureOpt"); - -// Public panel settings to set postprocessing shaders -const cmString CAL_OGLSELECTEDSWAPEFFECT("SwapEffect"); - -// Public panel settings to control CatalystAI settings -const cmString CAL_OGLCATALYSTAI("CatalystAI"); - - -// Public panel settings to set postprocessing shaders -const cmString CAL_OGLSUPPORTEDSWAPEFFECTS("SupportedSwapEffects"); - -const cmString CAL_OGLCUSTOMSWAPSOURCEFILE("CustomSwapSourceFile"); - -//Public panel setting for allowing special pixel shaders to be applied at swap time. -const cmString CAL_SPECIALSWAP("SpecialSwap"); - -//Public panel setting for special swap file -const cmString CAL_SPECIALSWAPFILE("SpecialSwapFile"); - - - -// Private Panel specific Defines -// - -// Private panel setting to force SW path -const cmString CAL_PICKSOFTWARE("PickSoftware"); - -// Private panel setting to force Microsoft path -const cmString CAL_PICKSOFTWAREMICROSOFT("PickSoftwareMicrosoft"); - -// Private Panel setting to enable TCL (versus forcing SW TCL) -const cmString CAL_ENABLETCL("EnableTCL"); - -// Private Panel setting to control HW Flips -const cmString CAL_ALLOWHWFLIP("AllowHWFlip"); - -// Private Panel setting to allow Z compression -const cmString CAL_ENABLEZCOMPRESSION("ZCompression"); - -// Private Panel setting to use fast z clears -const cmString CAL_ENABLEFASTZMASKCLEAR("FastZMaskClear"); - -// Private Panel setting to enable hierarchical Z -const cmString CAL_ENABLEHIERARCHICALZ("HierachicalZ"); - -// Private Panel setting to enable/disable cmask clears -const cmString CAL_ENABLECMASKCLEARS("MaskClears"); - -// Private Panel setting to force cmask clear after swap -const cmString CAL_CLEARCMASKAFTERSWAP("ClearCMaskAfterSwap"); - -// Private Panel setting to enable cmask compression -const cmString CAL_ENABLECMASKCOMPRESSION("CMaskCompression"); - -// Private Panel setting to force LOD Bias -const cmString CAL_LODBIAS("LODBias"); - -// Private Panel setting enable fast trilinear -const cmString CAL_FASTTRILINEAR("FastTrilinear"); - -// Private Panel setting to force clears to be skipped -const cmString CAL_DISABLECLEAR("DisableClear"); - -// Private Panel setting to control swapping -const cmString CAL_DISABLESWAP("DisableSwap"); - -// Private Panel setting to force HW idle after submit -const cmString CAL_WAITFORIDLEAFTERSUBMIT("WaitForIdleAfterSubmit"); - -// Private Panel setting to force single buffered rendering -const cmString CAL_FORCESINGLEBUFFER("ForceSingleBuffer"); - -// Private Panel setting to force buffer config for single buffered -// configs -const cmString CAL_SINGLE_BUF_CONFIG("SingleBufferConfig"); - -// Private Panel setting to force buffer config for double buffered -const cmString CAL_DOUBLE_BUF_CONFIG("DoubleBufferConfig"); - -// Private Panel setting to cause driver to breka on load -const cmString CAL_BREAK_ON_LOAD("BreakOnLoad"); - -// Private Panel setting for asserting when we set an error -const cmString CAL_ASSERTONERROR("AssertOnError"); - -// Private Panel setting to turn on shader dumping -const cmString CAL_ENABLESHADERDUMP("EnableShaderDump"); - -// Private Panel setting to turn on packet dumping -const cmString CAL_ENABLEPACKETDUMP("EnablePacketDump"); - -// Private Panel setting to set location of packet dump -const cmString CAL_PACKETDUMPLOCATION("PacketDumpLocation"); - -// Private Panel setting to set what type of file to be written -const cmString CAL_PACKETDUMPTYPE("PacketDumpType"); - -// Private Panel setting to select file overwrite -const cmString CAL_ONLYSAVELASTPACKET("OnlySaveLastPacket"); - -// Private Panel setting to turn on vcop patchlist dumping -const cmString CAL_ENABLEPATCHDUMP("EnablePatchDump"); - -// Private Panel setting to set dump file name -const cmString CAL_DUMPFILENAME("DumpFilename"); - -// Private Panel setting to control level of HW detail dumped -const cmString CAL_DUMPADDITIONALHWINFO("DumpAdditionalHWInfo"); - - -// Private Panel setting to select frames to dump -const cmString CAL_FRAMESTORECORD("FrameStoreCord"); - -// Private Panel setting to drop all PM4 packets -const cmString CAL_DROPFLUSH("DropFlush"); - -// Private Panel setting to furce use of dummy QS -const cmString CAL_ENABLEDUMMYQS("DummyQS"); - -// Private Panel setting to stub post setup -const cmString CAL_STUBPOSTSETUP("StubPostSetup"); - -// Private Panel setting to stub post TCL -const cmString CAL_STUBPOSTTCL("StubPostTCL"); - -// Private Panel setting to disable RB3D -const cmString CAL_DISABLERB3D("DisableR3D"); - -// Private Panel setting to disable alpha blend -const cmString CAL_DISABLEALPHABLEND("DisableAlphaBlend"); - -// Private Panel setting to force use of tiny textures -const cmString CAL_FORCETINYTEXTURES("ForceTinyTextures"); - -// Private Panel setting to prevent object allocation in AGP -const cmString CAL_OBJBUFINAGP("OBJBufferInAGP"); - -// Private Panel setting to prevent object allcoation in local -const cmString CAL_OBJBUFINLOCAL("OBJBufferInLocal"); - -// Private Panel setting to set the length of the swap queue -const cmString CAL_SWAPQUEUELENGTH("SwapQueueLength"); - -// Private Panel setting to enable macro tiling for textures -const cmString CAL_ENABLEMACROTILE("MacroTile"); - -// Private Panel setting to enable micro tiling for textures -const cmString CAL_ENABLEMICROTILE("MicroTile"); - -// Private Panel setting for allowing early z -const cmString CAL_ALLOWEARLYZ("AllowEarlyZ"); - -//Private Panel setting to allow for window to be broken into -// multiple pieces(allows full use of C and Z mask on R300 at high res); -const cmString CAL_ALLOWSPLITSCREEN("AllowSplitScreen"); - -//Private Panel setting for aniso threshold -const cmString CAL_ANISOTHRESHOLD("AnisoThreshold"); - -//Private Panel setting for aniso bias -const cmString CAL_ANISOLOD("AnisoLod"); - -//Private Panel setting fpr aniso bias -const cmString CAL_ANISOBIAS("AnisoBias"); - -//Private Panel setting to control ainos theshold mode -const cmString CAL_ANISOTHRESHMODE("AnisoThreshmode"); - -// Private Panel Setting for turnning off multi vpu mode(ie render everything to both) for the rest of a frame after a glCopyTexImage or glCopyTexSubImage happen. -const cmString CAL_DISABLEMVPUONCOPYTEX("DisableMVPUOnCopyTexture"); - -// Private Panel Setting for forcing swap to happen on slave vpu(useful for debugging); -const cmString CAL_FORCEMVPUSWAPONSLAVE("ForceMVPUSwapOnSlave"); - -// Private Panel Setting for skipping multi-vpu synchronization -const cmString CAL_SKIPMVPUSYNCH("SkipMVPUSynch"); - -// Private Panel Setting for controlling the percent of screen rendered on the master vpu -const cmString CAL_PERCENTONMASTERMVPU("PercentOnMasterMVPU"); - -// Private Panel Setting for controlling the mode of mvpu operation -const cmString CAL_MODEMVPU("ModeMVPU"); - -// Private Panel Setting for drawing a line where the scissored split happened in mvpu mode -const cmString CAL_DRAWSPLITLINEMVPU("DrawSplitLineMVPU"); - -// Private Panel Setting for controlling whether or not to unroll loops in the GLSL parser -const cmString CAL_UNROLL_LOOPS("UnrollLoops"); - -// Private Panel Spare setting 1 -const cmString CAL_SPARE1("Spare1"); - -// Private Panel Spare setting 2 -const cmString CAL_SPARE2("Spare2"); - -// Private Panel Spare setting 3 -const cmString CAL_SPARE3("Spare3"); - -// Private Panel Spare setting 4 -const cmString CAL_SPARE4("Spare4"); - -// Private Panel Spare setting 5 -const cmString CAL_SPARE5("Spare5"); - -// Private Panel Spare setting 6 -const cmString CAL_SPARE6("Spare6"); - -// Private Panel Spare setting 7 -const cmString CAL_SPARE7("Spare7"); - -// Private Panel Spare setting 8 -const cmString CAL_SPARE8("Spare8"); - -// Private Panel Spare setting 9 -const cmString CAL_SPARE9("Spare9"); - -// Private Panel Spare setting 10 -const cmString CAL_SPARE10("Spare10"); - -// Private Panel Spare setting 11 - accepts numbers, not just 0 and 1 -const cmString CAL_SPARE11("Spare11"); - -// Private Panel Spare setting 12 - accepts numbers, not just 0 and 1 -const cmString CAL_SPARE12("Spare12"); - -// Private Panel Spare setting 12 - accepts numbers, not just 0 and 1 -const cmString CAL_PS3ENABLE("PS3Enable"); - -// Private Panel setting for asserting when we punt to SW -const cmString CAL_ASSERTONSWPUNT("OrcaAssertOnSWPunt"); - -// Private Panel setting for logging when we punt to SW -const cmString CAL_LOGSWPUNTCASES("OrcaLogSWPuntCases"); - -// Private Panel setting to set punt log file name -const cmString CAL_PUNTLOGFILENAME("OrcaPuntLogFileName"); - -// softVAP mode -const cmString CAL_SOFTVAP("SoftVAP"); - -// softVAP il compile mode -const cmString CAL_SVPOFFLINECOMPILE("SvpOfflineCompile"); - -const cmString CAL_EMULATOR("Emulator"); - -const cmString CAL_ENABLE_FORCE_ASIC_ID("EnableForceAsicID"); -const cmString CAL_FORCE_ASIC_ID("ForceAsicID"); -const cmString CAL_FORCE_REMOTE_MEMORY("ForceRemoteMemory"); -const cmString CAL_DISABLE_ASYNC_DMA("DisableAsyncDma"); -const cmString CAL_ENABLE_DUMP_IL("DumpIL"); -const cmString CAL_ENABLE_DUMP_ISA("DumpISA"); - -// TDR -const cmString CAL_ENABLE_FLUSH_AFTER_RENDER("FlushAfterRender"); - -// VM Disabling -const cmString CAL_DISABLE_VM("DisableVM"); - -#endif - - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.cpp deleted file mode 100644 index 250d416ee3..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.cpp +++ /dev/null @@ -1,556 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -// -// Trade secret of ATI Technologies, Inc. -// Copyright 2005, ATI Technologies, Inc., (unpublished) -// -// All rights reserved. This notice is intended as a precaution against -// inadvertent publication and does not imply publication or any waiver -// of confidentiality. The year included in the foregoing notice is the -// year of creation of the work. -// - -/// @file inifile.cpp -/// @brief INI File Parser - -#include "inifile.h" -#include "cm_string.h" -#include "inifile_parser.h" -#include "assert.h" - -#include -#include -#include - -#ifdef DEBUG - #include - #include -#endif - -/** - * IniValueString members - */ - -IniValueString::IniValueString() -{ - value = cmString(""); -} - -IniValueString::IniValueString(const IniValueString& val) -{ - value = val.value; -} - -IniValueString::IniValueString(cmString val) -{ - value = val; -} - -IniValueString& IniValueString::operator=(IniValueString& v) -{ - value = v.value; - return *this; -} - -CALboolean IniValueString::getValue(cmString* value) -{ - *value = this->value; - return CAL_TRUE; -} - - -/** - * IniValueBool members - */ - -IniValueBool::IniValueBool() -{ - value = CAL_FALSE; -} - -IniValueBool::IniValueBool(const IniValueBool& val) -{ - value = val.value; -} - -IniValueBool::IniValueBool(CALboolean val) -{ - value = val; -} - -IniValueBool& IniValueBool::operator=(IniValueBool& v) -{ - value = v.value; - return *this; -} - -CALboolean IniValueBool::getValue(CALboolean* value) -{ - *value = this->value; - return CAL_TRUE; -} - - -/** - * IniValueInt members - */ - -IniValueInt::IniValueInt() -{ - value = 0; -} - -IniValueInt::IniValueInt(const IniValueInt& val) -{ - value = val.value; -} - -IniValueInt::IniValueInt(CALint val) -{ - value = val; -} - -IniValueInt& IniValueInt::operator=(IniValueInt& v) -{ - value = v.value; - return *this; -} - -CALboolean IniValueInt::getValue(CALint* value) -{ - *value = this->value; - return CAL_TRUE; -} - -/** - * IniValueFloat members - */ - -IniValueFloat::IniValueFloat() -{ - value = 0; -} - -IniValueFloat::IniValueFloat(const IniValueFloat& val) -{ - value = val.value; -} - -IniValueFloat::IniValueFloat(CALfloat val) -{ - value = val; -} - -IniValueFloat& IniValueFloat::operator=(IniValueFloat& v) -{ - value = v.value; - return *this; -} - -CALboolean IniValueFloat::getValue(CALfloat* value) -{ - *value = this->value; - return CAL_TRUE; -} - -/** - * IniSection Members - */ -IniSection::IniSection() -{ - name = cmString(""); -} - -IniSection::IniSection(const IniSection& s) -{ - name = s.name; - for(EntryDBIterator iter = s.entryDB.begin() ; iter != s.entryDB.end(); ++iter) - { - entryDB[iter->first] = iter->second; - } - -} - -IniSection::IniSection(cmString n) -{ - name = n; -} - -IniSection::~IniSection() -{ - for(EntryDBIterator iter = entryDB.begin() ; iter != entryDB.end(); ++iter) - { - delete iter->second; - } - entryDB.clear(); -} - -IniSection& IniSection::operator=(IniSection& s) -{ - name = s.name;; - entryDB.clear(); - for(EntryDBIterator iter = s.entryDB.begin() ; iter != s.entryDB.end(); iter++) - { - entryDB[iter->first] = iter->second; - } - return *this; -} - - -void IniSection::addEntry(cmString name, IniValue* value) -{ - IniValue* v = findEntry(name); - if (v) - { - delete v; - } - entryDB[name] = value; -} - -IniValue* IniSection::findEntry(cmString name) -{ - EntryDBIterator iter = entryDB.find(name); - if(iter != entryDB.end()) - { - return iter->second; - } - else - { - return NULL; - } - -} - -/** - * IniFile members - */ - -IniFile::IniFile(cmString filename) -{ - #ifdef DEBUG - SanityTest(); - #endif - - std::ifstream in(filename.c_str()); - IniFileParser::Parse(in, *this); -} - -IniFile::IniFile(std::istream& in) -{ - IniFileParser::Parse(in, *this); -} - -IniFile::~IniFile() -{ - for(SectionDBIterator iter = sectionDB.begin() ; iter != sectionDB.end(); iter++) - { - delete iter->second; - } - sectionDB.clear(); -} - -const cmString IniSection::getName () const -{ - return name; -} - -void IniFile::addSection(IniSection* section) -{ - IniSection* v = findSection(section->getName()); - if (v) - { - delete v; - } - sectionDB[section->getName()] = section; -} - -IniSection* IniFile::findSection(cmString section) -{ - SectionDBIterator iter = sectionDB.find(section); - if (iter != sectionDB.end()) - { - return iter->second; - } - else - { - return NULL; - } -} - -IniValue* IniFile::getValue(cmString section, cmString entry) -{ - IniSection* s = findSection(section); - if(s == NULL) - { - return NULL; - } - - return s->findEntry(entry); -} - - -CALboolean IniFile::getValue(cmString section, cmString entry, CALboolean* value) -{ - IniValue* v = getValue(section, entry); - if (v != NULL) - { - return v->getValue(value); - } - return CAL_FALSE; -} - -CALboolean IniFile::getValue(cmString section, cmString entry, CALint* value) -{ - IniValue* v = getValue(section, entry); - if (v != NULL) - { - return v->getValue(value); - } - return CAL_FALSE; -} - -CALboolean IniFile::getValue(cmString section, cmString entry, CALfloat* value) -{ - IniValue* v = getValue(section, entry); - if (v != NULL) - { - return v->getValue(value); - } - return CAL_FALSE; -} - -CALboolean IniFile::getValue(cmString section, cmString entry, cmString* value) -{ - IniValue* v = getValue(section, entry); - if (v != NULL) - { - return v->getValue(value); - } - return CAL_FALSE; -} - -/** - * Debug only methods - * - */ -#ifdef DEBUG - -void IniValueString::printAST() -{ - std::cerr << value.c_str() << " [string]\n"; -} - -void IniValueBool::printAST() -{ - std::cerr << value << " [bool]\n"; -} - - -void IniValueInt::printAST() -{ - std::cerr << value << " [int]\n"; -} - -void IniValueFloat::printAST() -{ - std::cerr << value << " [float]\n"; -} - -void IniSection::printAST() -{ - for(EntryDBIterator iter = entryDB.begin() ; iter != entryDB.end(); ++iter) - { - cmString name = iter->first; - IniValue *v = iter->second; - - std::cerr << name.c_str() << " = "; - v->printAST(); - } -} - - -void IniFile::printAST() -{ - for(SectionDBIterator iter = sectionDB.begin() ; iter != sectionDB.end(); ++iter) - { - IniSection* s = iter->second; - std::cerr << "[" << s->getName().c_str() << "]\n"; - s->printAST(); - } - std::cerr << "\n"; -} - -void IniFile::SanityTest() -{ - //std::cerr << "Running IniFile Sanity...\n"; - - static const cmString section("section"); - - - static const std::string file1( -"[section]\n\ -bool1=true\n\ -bool2=false\n\ -int=3\n\ -float=1.1111\n\ -string=abc def\n"); - - std::istringstream s1(file1); - IniFile* iniFile = new IniFile(s1); - //iniFile->printAST(); - - CALboolean b; - - assert(iniFile->getValue(section, cmString("bool1"), &b) == CAL_TRUE); - assert(b == CAL_TRUE); - - assert(iniFile->getValue(section, cmString("bool2"), &b) == CAL_TRUE); - assert(b == CAL_FALSE); - - CALint i; - assert(iniFile->getValue(section, cmString("int"), &i) == CAL_TRUE); - assert(i == 3); - - - CALfloat f; - assert(iniFile->getValue(section, cmString("float"), &f) == CAL_TRUE); - assert(f == 1.1111f); - - cmString s; - assert(iniFile->getValue(section, cmString("string"), &s) == CAL_TRUE); - assert(s == cmString("abc def")); - - i = -1; - // Wrong section - assert(iniFile->getValue(cmString("dummy"), cmString("int"), &i) == CAL_FALSE); - assert(i == -1); - - // Wrong entry - assert(iniFile->getValue(section, cmString("dummy"), &i) == CAL_FALSE); - assert(i == -1); - - - static const std::string file2( -"[section]\n\ -bool1=1true\n\ -bool2=false2\n\ -int=3a\n\ -float=1.1111b\n\ -string=1\n"); - - delete iniFile; - - std::istringstream s2(file2); - iniFile = new IniFile(s2); - //iniFile->printAST(); - - cmString str; - - b = CAL_FALSE; - // try to get a bool, then a string - assert(iniFile->getValue(section, cmString("bool1"), &b) == CAL_FALSE); - assert(b == CAL_FALSE); - assert(iniFile->getValue(section, cmString("bool1"), &str) == CAL_TRUE); - assert(str == cmString("1true")); - - // try to get a bool, then a string - assert(iniFile->getValue(section, cmString("bool2"), &b) == CAL_FALSE); - assert(b == CAL_FALSE); - assert(iniFile->getValue(section, cmString("bool2"), &str) == CAL_TRUE); - assert(str == cmString("false2")); - - i = -1; - // try to get an int, then a string - assert(iniFile->getValue(section, cmString("int"), &i) == CAL_FALSE); - assert(i == -1); - assert(iniFile->getValue(section, cmString("int"), &str) == CAL_TRUE); - assert(str == cmString("3a")); - - - f = -1.1f; - // try to get a float, then a string - assert(iniFile->getValue(section, cmString("float"), &f) == CAL_FALSE); - assert(f == -1.1f); - assert(iniFile->getValue(section, cmString("float"), &str) == CAL_TRUE); - assert(str == cmString("1.1111b")); - - // try to get a string, value is an int - assert(iniFile->getValue(section, cmString("string"), &str) == CAL_FALSE); - assert(str == cmString("1.1111b")); - assert(iniFile->getValue(section, cmString("string"), &i) == CAL_TRUE); - assert(i == 1); - - static const cmString section1("section1"); - static const cmString section2("section2"); - static const cmString section3("section3"); - - static const std::string file3( -"[section1\n\ -bool1=false\n\ -bool2=false\n\ -int=1\n\ -float=1.1\n\ -string=abc\n\ -[section2]\n\ -bool1=true\n\ -bool2=true\n\ -int=2\n\ -float=1.2\n\ -string=def\n\ -[section3]\n\ -int=3\n\ -[section2]\n\ -float=1.3\n"); - - delete iniFile; - - std::istringstream s3(file3); - iniFile = new IniFile(s3); - //iniFile->printAST(); - - // section1 should not exist (syntax error) - assert(iniFile->getValue(section1, cmString("bool1"), &str) == CAL_FALSE); - assert(iniFile->getValue(section1, cmString("bool2"), &str) == CAL_FALSE); - assert(iniFile->getValue(section1, cmString("int"), &str) == CAL_FALSE); - assert(iniFile->getValue(section1, cmString("float"), &str) == CAL_FALSE); - assert(iniFile->getValue(section1, cmString("string"), &str) == CAL_FALSE); - - // section2 should exist, only with the float - assert(iniFile->getValue(section2, cmString("bool1"), &b) == CAL_FALSE); - assert(iniFile->getValue(section2, cmString("bool2"), &b) == CAL_FALSE); - assert(iniFile->getValue(section2, cmString("int"), &i) == CAL_FALSE); - - // overridden - assert(iniFile->getValue(section2, cmString("float"), &f) == CAL_TRUE); - assert(f == 1.3f); - - assert(iniFile->getValue(section2, cmString("string"), &str) == CAL_FALSE); - - // section3 had a differant int - assert(iniFile->getValue(section3, cmString("int"), &i) == CAL_TRUE); - assert(i == 3); - - delete iniFile; - - //std::cerr << "Done!"; -} - -#endif diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.h deleted file mode 100644 index a44d36d9ad..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile.h +++ /dev/null @@ -1,184 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef INIFILE_H -#define INIFILE_H -// -// Trade secret of ATI Technologies, Inc. -// Copyright 2005, ATI Technologies, Inc., (unpublished) -// -// All rights reserved. This notice is intended as a precaution against -// inadvertent publication and does not imply publication or any waiver -// of confidentiality. The year included in the foregoing notice is the -// year of creation of the work. -// - -/// @file inifile.h -/// @brief INI File Parser - -#include "cm_string.h" -#include "backend.h" - -#include -#include - -class IniValue -{ -public: - virtual ~IniValue() {} - virtual CALboolean getValue(CALboolean* value) { return CAL_FALSE; }; - virtual CALboolean getValue(CALint* value) { return CAL_FALSE; }; - virtual CALboolean getValue(CALfloat* value) { return CAL_FALSE; }; - virtual CALboolean getValue(cmString* value) { return CAL_FALSE; }; - -#ifdef DEBUG - virtual void printAST() {}; -#endif -private: - -}; - -class IniValueBool : public IniValue -{ -public: - IniValueBool(); - IniValueBool(const IniValueBool& val); - IniValueBool(CALboolean val); - IniValueBool& operator=(IniValueBool& v); - - CALboolean getValue(CALboolean* value); - -#ifdef DEBUG - void printAST(); -#endif -private: - CALboolean value; -}; - -class IniValueString : public IniValue -{ -public: - IniValueString(); - IniValueString(const IniValueString& val); - IniValueString(cmString val); - IniValueString& operator=(IniValueString& v); - - CALboolean getValue(cmString* value); - -#ifdef DEBUG - void printAST(); -#endif -private: - cmString value; -}; - -class IniValueInt : public IniValue -{ -public: - IniValueInt(); - IniValueInt(const IniValueInt& val); - IniValueInt(CALint val); - IniValueInt& operator=(IniValueInt& v); - - CALboolean getValue(CALint* value); - void printAST(); -private: - CALint value; -}; - -class IniValueFloat : public IniValue -{ -public: - IniValueFloat(); - IniValueFloat(const IniValueFloat& val); - IniValueFloat(CALfloat val); - IniValueFloat& operator=(IniValueFloat& v); - - CALboolean getValue(CALfloat* value); - -#ifdef DEBUG - void printAST(); -#endif -private: - CALfloat value; -}; - - - -class IniSection -{ -public: - IniSection(); - IniSection(const IniSection& s); - IniSection(cmString n); - ~IniSection(); - - IniSection& operator=(IniSection& s); - - void addEntry(cmString name, IniValue* value); - IniValue* findEntry(cmString name); - const cmString getName(); -#ifdef DEBUG - void printAST(); -#endif -private: - typedef std::map EntryDB; - typedef EntryDB::const_iterator EntryDBIterator; - typedef std::pair EntryDBPair; - - cmString name; - EntryDB entryDB; -}; - - - -class IniFile -{ -public: - IniFile(cmString filename); - IniFile(std::istream& in); - ~IniFile(); - - CALboolean getValue(cmString section, cmString entry, CALboolean* value); - CALboolean getValue(cmString section, cmString entry, CALint* value); - CALboolean getValue(cmString section, cmString entry, CALfloat* value); - CALboolean getValue(cmString section, cmString entry, cmString* value); - - // should be protected - void addSection(IniSection* section); - IniSection* findSection(cmString section); -#ifdef DEBUG - void printAST(); - static void SanityTest(); -#endif -private: - typedef std::map SectionDB; - typedef SectionDB::const_iterator SectionDBIterator; - typedef std::pair SectionDBPair; - - IniValue* getValue(cmString section, cmString entry); - - - SectionDB sectionDB; -}; - - - -#endif diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.cpp b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.cpp deleted file mode 100644 index 3059d15332..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.cpp +++ /dev/null @@ -1,245 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -// -// Trade secret of ATI Technologies, Inc. -// Copyright 2005, ATI Technologies, Inc., (unpublished) -// -// All rights reserved. This notice is intended as a precaution against -// inadvertent publication and does not imply publication or any waiver -// of confidentiality. The year included in the foregoing notice is the -// year of creation of the work. -// - -/// @file inifile_parser.cpp -/// @brief INI File Parser Implementation - -#include "inifile.h" -#include "inifile_parser.h" - -#include "cm_string.h" - - -#include -#include -#include -#include -#include -#include -#include -#include - - -void IniFileParser::Parse(std::istream& in, IniFile& iniFile) -{ - CALuint count = 0; - std::string line; - bool inSection = false; - - std::string sectionName; - - IniSection* section = NULL; - while(std::getline(in, line)) { - count++; - cleanup(line); - if(line.empty()) - { - continue; - } - if(parseSectionName(line, sectionName)) - { - section = new IniSection(cmString(sectionName.c_str())); - iniFile.addSection(section); - inSection = true; - } - else if(inSection) - { - parseLine(line, section, count); - } - } -} - -void IniFileParser::parseLine( std::string line, IniSection* section, CALuint count ) { - std::string::size_type equals = line.find( '=' ); - if ( equals == std::string::npos ) { -#ifdef DEBUG - std::cerr << "IniFileParser: Could not parse line " << count << ", ignoring.\n"; -#endif - return; - } - - std::string name( line, 0, equals ); - IniValue* value = parseValue( std::string( line, equals + 1, std::string::npos)); - - section->addEntry(cmString(trim(name).c_str()), value); -} - -void IniFileParser::cleanup( std::string& line ) { - std::string copy = line; - unsigned int begin = 0; - while ( begin != line.size() && isspace(line[begin])) - { - ++begin; - } - - bool inQuote = false; - unsigned int end; - for(end = begin; end != line.size(); ++end) - { - if ( line[end] == '\"' ) - { - inQuote = !inQuote; - } - // comments starts with # or ; - else if ( (line[end] == '#' || line[end] == ';') && !inQuote ) - { - break; - } - else if ( line[ end ] == '\\' ) - { - ++end; // ignore next character - if ( end == line.size() ) { -#ifdef DEBUG - std::cerr << "INIFileParser: Error parsing file: \\ character " - "at the end of line (sorry, not supported)\n"; -#endif - break; - } - } - } - while ( end > begin && isspace( line[ end - 1 ] ) ) --end; - // This is used over assign so that we don't have memcpy overrun - // errors in valgrind. - line = line.substr(begin, end - begin); -} - -class isint -{ -public: - isint() - { - is_int = true; - } - void operator() (char c) - { - is_int = is_int && isdigit(c); - } - bool is_int; -}; - -class isfloat -{ -public: - isfloat() - { - is_float = true; - } - void operator() (char c) - { - is_float = is_float && (isdigit(c) || c == '.'); - } - bool is_float; -}; - -int cmp_nocase(const std::string& s1, const std::string& s2) -{ - std::string::const_iterator p1 = s1.begin(); - std::string::const_iterator p2 = s2.begin(); - - while( p1 != s1.end() && p2 != s2.end()) - { - if(toupper(*p1) != toupper(*p2)) - { - return (toupper(*p1) < toupper(*p2)) ? -1 : 1; - } - ++p1; - ++p2; - } - return static_cast(s2.size()-s1.size()); -} - -IniValue* IniFileParser::parseValue(std::string& value ) { - std::string trimmed = trim(value); - - std::stringstream ss(trimmed); - - // look for a boolean - static const std::string strTrue("true"); - static const std::string strFalse("false"); - if(cmp_nocase(trimmed, strTrue) == 0) - { - return new IniValueBool(CAL_TRUE); - } - if(cmp_nocase(trimmed, strFalse) == 0) - { - return new IniValueBool(CAL_FALSE); - } - - // try now to get an int - isint ii; - ii = std::for_each(trimmed.begin(),trimmed.end(), ii); - if(ii.is_int) - { - CALint intValue = 0; - ss >> intValue; - return new IniValueInt(intValue); - } - - // if not an int, try to get a float - isfloat isf; - isf = std::for_each(trimmed.begin(),trimmed.end(), isf); - if(isf.is_float) - { - CALfloat floatValue; - // mbeuchat: Remove STL conversion of string to float. When compiled - // on Linux, DK g++ with optimization requires linking against - // libstdc++-6.0.9 which is not available on all Linux systems. - // ss >> floatValue; - floatValue = (float)atof(ss.str().c_str()); - return new IniValueFloat(floatValue); - } - - // finally, default to a string - return new IniValueString(cmString(trimmed.c_str())); -} - -bool IniFileParser::parseSectionName(std::string line, std::string& section ) -{ - if ( line[ 0 ] != '[' ) return false; - if ( line[ line.size() - 1 ] != ']' ) return false; - - section.assign( line, 1, line.size() - 2 ); - return true; -} - - -std::string IniFileParser::trim(std::string const& source, char const* delims) { - std::string result(source); - std::string::size_type index = result.find_last_not_of(delims); - if(index != std::string::npos) - result.erase(++index); - - index = result.find_first_not_of(delims); - if(index != std::string::npos) - result.erase(0, index); - else - result.erase(); - return result; -} diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.h b/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.h deleted file mode 100644 index d8faad0fe6..0000000000 --- a/projects/clr/rocclr/device/gpu/gslbe/src/rt/inifile/inifile_parser.h +++ /dev/null @@ -1,61 +0,0 @@ - /* Copyright (c) 2008 - 2021 Advanced Micro Devices, Inc. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. */ - -#ifndef INIFILE_PARSER_H -#define INIFILE_PARSER_H -// -// Trade secret of ATI Technologies, Inc. -// Copyright 2005, ATI Technologies, Inc., (unpublished) -// -// All rights reserved. This notice is intended as a precaution against -// inadvertent publication and does not imply publication or any waiver -// of confidentiality. The year included in the foregoing notice is the -// year of creation of the work. -// - -/// @file inifile_parser.h -/// @brief INI File Parser Implementation - -// if compiled from OGTST, add the following, normally defined in atitypes.h - -#include "inifile.h" -#include "cm_string.h" - -#include -#include -#include - -class IniFileParser -{ -public: - static void Parse(std::istream& in, IniFile& iniFile); - -private: - static void parseLine( std::string line, IniSection* section, CALuint count ); - static bool parseSectionName(std::string line, std::string& section ); - static IniValue* parseValue(std::string value ); - static void cleanup( std::string& line ); - static std::string trim(std::string const& source, char const* delims = " \t\r\n"); -}; - -#endif - - - diff --git a/projects/clr/rocclr/device/gpu/gslbe/src/rt/DxxOpenCLInteropExt.h b/projects/clr/rocclr/device/pal/DxxOpenCLInteropExt.h similarity index 100% rename from projects/clr/rocclr/device/gpu/gslbe/src/rt/DxxOpenCLInteropExt.h rename to projects/clr/rocclr/device/pal/DxxOpenCLInteropExt.h diff --git a/projects/clr/rocclr/device/pal/paldefs.hpp b/projects/clr/rocclr/device/pal/paldefs.hpp index 19e15b325e..68c36873e4 100644 --- a/projects/clr/rocclr/device/pal/paldefs.hpp +++ b/projects/clr/rocclr/device/pal/paldefs.hpp @@ -307,36 +307,7 @@ static constexpr MemoryFormat MemoryFormatMap[] = { Pal::ChNumFormat::X32Y32_Float, {Pal::ChannelSwizzle::X, Pal::ChannelSwizzle::Y, Pal::ChannelSwizzle::Zero, Pal::ChannelSwizzle::One}}, - /* - // RA - { { CL_RA, CL_UNORM_INT8 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG8 } }, - { { CL_RA, CL_UNORM_INT16 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG16 } }, - { { CL_RA, CL_SNORM_INT8 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG8 } }, - { { CL_RA, CL_SNORM_INT16 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sUV16 } }, - - { { CL_RA, CL_SIGNED_INT8 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG8I } }, - { { CL_RA, CL_SIGNED_INT16 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG16I } }, - { { CL_RA, CL_SIGNED_INT32}, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_sRG32I } }, - { { CL_RA, CL_UNSIGNED_INT8 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG8I } }, - { { CL_RA, CL_UNSIGNED_INT16 }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG16I } }, - { { CL_RA, CL_UNSIGNED_INT32}, - { GSL_CHANNEL_ORDER_RA , CM_SURF_FMT_RG32I } }, - - { { CL_RA, CL_HALF_FLOAT }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG16F } }, - { { CL_RA, CL_FLOAT }, - { GSL_CHANNEL_ORDER_RA, CM_SURF_FMT_RG32F } }, - */ // RGB {{CL_RGB, CL_UNORM_INT_101010}, Pal::ChNumFormat::X10Y10Z10W2_Unorm, diff --git a/projects/clr/rocclr/device/pal/paldevice.cpp b/projects/clr/rocclr/device/pal/paldevice.cpp index ecc1edd1c2..4052047a1f 100644 --- a/projects/clr/rocclr/device/pal/paldevice.cpp +++ b/projects/clr/rocclr/device/pal/paldevice.cpp @@ -1755,14 +1755,14 @@ bool Device::bindExternalDevice(uint flags, void* const pDevice[], void* pContex #ifdef _WIN32 if (flags & amd::Context::Flags::D3D10DeviceKhr) { if (!associateD3D10Device(pDevice[amd::Context::DeviceFlagIdx::D3D10DeviceKhrIdx])) { - LogError("Failed gslD3D10Associate()"); + LogError("Failed associateD3D10Device()"); return false; } } if (flags & amd::Context::Flags::D3D11DeviceKhr) { if (!associateD3D11Device(pDevice[amd::Context::DeviceFlagIdx::D3D11DeviceKhrIdx])) { - LogError("Failed gslD3D11Associate()"); + LogError("Failed associateD3D11Device()"); return false; } } @@ -1783,10 +1783,10 @@ bool Device::bindExternalDevice(uint flags, void* const pDevice[], void* pContex #endif //_WIN32 if (flags & amd::Context::Flags::GLDeviceKhr) { - // Attempt to associate GSL-OGL + // Attempt to associate PAL-OGL if (!glAssociate(pContext, pDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx])) { if (!validateOnly) { - LogError("Failed gslGLAssociate()"); + LogError("Failed glAssociate()"); } return false; } @@ -1803,10 +1803,10 @@ bool Device::unbindExternalDevice(uint flags, void* const pDevice[], void* pCont void* glDevice = pDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx]; if (glDevice != nullptr) { - // Dissociate GSL-OGL + // Dissociate PAL-OGL if (!glDissociate(pContext, glDevice)) { if (validateOnly) { - LogWarning("Failed gslGLDiassociate()"); + LogWarning("Failed glDissociate()"); } return false; } diff --git a/projects/clr/rocclr/device/pal/paldevice.hpp b/projects/clr/rocclr/device/pal/paldevice.hpp index e9f1adf5de..90f9cb4f4b 100644 --- a/projects/clr/rocclr/device/pal/paldevice.hpp +++ b/projects/clr/rocclr/device/pal/paldevice.hpp @@ -491,7 +491,7 @@ class Device : public NullDevice { //! Initialize heap resources if uninitialized bool initializeHeapResources(); - //! Set GSL sampler to the specified state + //! Set HW sampler to the specified state void fillHwSampler(uint32_t state, //!< Sampler's OpenCL state void* hwState, //!< Sampler's HW state uint32_t hwStateSize, //!< Size of sampler's HW state diff --git a/projects/clr/rocclr/device/pal/palresource.cpp b/projects/clr/rocclr/device/pal/palresource.cpp index 8e47100b26..95363b5382 100644 --- a/projects/clr/rocclr/device/pal/palresource.cpp +++ b/projects/clr/rocclr/device/pal/palresource.cpp @@ -980,7 +980,7 @@ bool Resource::CreateInterop(CreateParams* params) { dev().iDev()->CreateImageViewSrds(1, &viewInfo, hwState_); //! It's a workaround for D24S8 format, since PAL doesn't support this format - //! and GSL decompresses 24bit DEPTH into D24S8 for OGL compatibility + //! and OGL decompresses 24bit DEPTH into D24S8 for OGL compatibility if ((desc().format_.image_channel_order == CL_DEPTH_STENCIL) && (desc().format_.image_channel_data_type == CL_UNORM_INT24)) { if (dev().settings().gfx10Plus_) { diff --git a/projects/clr/rocclr/device/rocm/rocsettings.cpp b/projects/clr/rocclr/device/rocm/rocsettings.cpp index 7b81398e2c..04d0008dcb 100644 --- a/projects/clr/rocclr/device/rocm/rocsettings.cpp +++ b/projects/clr/rocclr/device/rocm/rocsettings.cpp @@ -18,7 +18,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef WITHOUT_GPU_BACKEND +#ifndef WITHOUT_HSA_BACKEND #include "top.hpp" #include "os/os.hpp" @@ -232,4 +232,4 @@ void Settings::override() { } } // namespace roc -#endif // WITHOUT_GPU_BACKEND +#endif // WITHOUT_HSA_BACKEND diff --git a/projects/clr/rocclr/utils/flags.cpp b/projects/clr/rocclr/utils/flags.cpp index bb4361742b..616743d534 100644 --- a/projects/clr/rocclr/utils/flags.cpp +++ b/projects/clr/rocclr/utils/flags.cpp @@ -84,15 +84,11 @@ namespace amd { bool IS_HIP = false; std::atomic_bool IS_PROFILER_ON(false); -#if defined(WITH_GPU_DEVICE) -bool IS_LEGACY = true; -#else #if !defined(_WIN32) && defined(WITH_PAL_DEVICE) bool IS_LEGACY = true; #else bool IS_LEGACY = false; #endif -#endif // static char* Flag::envstr_; diff --git a/projects/clr/rocclr/utils/flags.hpp b/projects/clr/rocclr/utils/flags.hpp index 8f0228cc95..12487cb598 100644 --- a/projects/clr/rocclr/utils/flags.hpp +++ b/projects/clr/rocclr/utils/flags.hpp @@ -143,7 +143,7 @@ release(bool, GPU_IFH_MODE, false, \ release(bool, GPU_MIPMAP, true, \ "Enables GPU mipmap extension") \ release(uint, GPU_ENABLE_PAL, 2, \ - "Enables PAL backend. 0 - GSL(default), 1 - PAL, 2 - GSL and PAL") \ + "Enables PAL backend. 0 - ROC, 1 - PAL, 2 - ROC or PAL") \ release(bool, DISABLE_DEFERRED_ALLOC, false, \ "Disables deferred memory allocation on device") \ release(int, AMD_GPU_FORCE_SINGLE_FP_DENORM, -1, \