Arm Patinyasakdikul
1ce83d5cc0
* Added ERROR message class to handle fatal error messages. New ERROR message class will print the message in all debug level, including none. Change some of the fatal error message to be in ERROR instead of WARN. Added new error handler function to print out more meaningful error message in the future. * Added CHANGELOG entry. * Update CHANGELOG.md Co-authored-by: Jeffrey Novotny <jnovotny@amd.com> * Change to no longer reuse NONE as ERROR. ERROR is now a separated class. * Update CHANGELOG.md Co-authored-by: Jeffrey Novotny <jnovotny@amd.com> --------- Co-authored-by: Jeffrey Novotny <jnovotny@amd.com>
63 строки
2.0 KiB
C
63 строки
2.0 KiB
C
/*************************************************************************
|
|
* Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved.
|
|
*
|
|
* See LICENSE.txt for license information
|
|
************************************************************************/
|
|
|
|
#ifndef NCCL_INT_DEBUG_H_
|
|
#define NCCL_INT_DEBUG_H_
|
|
|
|
#include "nccl.h"
|
|
#include "nccl_common.h"
|
|
#include <stdio.h>
|
|
|
|
#include <pthread.h>
|
|
|
|
// Conform to pthread and NVTX standard
|
|
#define NCCL_THREAD_NAMELEN 16
|
|
|
|
extern int ncclDebugLevel;
|
|
extern FILE *ncclDebugFile;
|
|
|
|
void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...) __attribute__ ((format (printf, 5, 6)));
|
|
|
|
// Let code temporarily downgrade WARN into INFO
|
|
extern thread_local int ncclDebugNoWarn;
|
|
extern char ncclLastError[];
|
|
|
|
#define ERROR(...) ncclDebugLog(NCCL_LOG_ERROR, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__)
|
|
#define VERSION(...) ncclDebugLog(NCCL_LOG_VERSION, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__)
|
|
#define WARN(...) ncclDebugLog(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__)
|
|
#define INFO(FLAGS, ...) ncclDebugLog(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__)
|
|
#define TRACE_CALL(...) ncclDebugLog(NCCL_LOG_TRACE, NCCL_CALL, __func__, __LINE__, __VA_ARGS__)
|
|
|
|
#ifdef ENABLE_TRACE
|
|
#define TRACE(FLAGS, ...) ncclDebugLog(NCCL_LOG_TRACE, (FLAGS), __func__, __LINE__, __VA_ARGS__)
|
|
#else
|
|
#define TRACE(...)
|
|
#endif
|
|
|
|
void ncclSetThreadName(pthread_t thread, const char *fmt, ...);
|
|
|
|
void ncclResetDebugInit();
|
|
|
|
// RCCL custom error message handling.
|
|
static inline ncclResult_t rcclCudaErrorHandler(cudaError_t err) {
|
|
|
|
// Print the cuda error
|
|
ERROR("HIP failure: '%s'", cudaGetErrorString(err));
|
|
|
|
// Special error message here:
|
|
switch (err) {
|
|
case cudaErrorStreamCaptureInvalidated:
|
|
ERROR("Application is trying to use an invalidated stream to launch RCCL kernel. "
|
|
"This operation is invalid. RCCL is exiting.");
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
return ncclUnhandledCudaError;
|
|
}
|
|
|
|
#endif
|