From a8a5960095daf91a9804d4f667946969ca85210b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 17 Mar 2016 18:45:40 -0400 Subject: [PATCH] Add environment variable to disable GPU caching This option may help debug synchronization or coherency issues involving the GPU caches. It works only on dGPUs, by changing the cache policy of the GPUVM default aperture to "cohrent", which is implemented as non-cached on current dGPU hardware. Change-Id: I544ac9cc5c0cf1fa5c4e30f67aa42b3b5e44ae67 [ROCm/ROCR-Runtime commit: 06d391c6c95e0c420370f4b5c98b5655fcc6c7a4] --- projects/rocr-runtime/src/fmm.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/projects/rocr-runtime/src/fmm.c b/projects/rocr-runtime/src/fmm.c index ae1a5f7aab..5d739eaa5c 100644 --- a/projects/rocr-runtime/src/fmm.c +++ b/projects/rocr-runtime/src/fmm.c @@ -1052,6 +1052,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HsaNodeProperties props; struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + char *disableCache; + + /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ + disableCache = getenv("HSA_DISABLE_CACHE"); + if (disableCache && strcmp(disableCache, "0") == 0) + disableCache = NULL; /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than @@ -1180,9 +1186,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, - KFD_IOC_CACHE_POLICY_NONCOHERENT, - KFD_IOC_CACHE_POLICY_COHERENT, - alt_base, alt_size); + disableCache ? + KFD_IOC_CACHE_POLICY_COHERENT : + KFD_IOC_CACHE_POLICY_NONCOHERENT, + KFD_IOC_CACHE_POLICY_COHERENT, + alt_base, alt_size); if (err != 0) { fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR;