revert npkit (#748)
This commit is contained in:
Ziyue Yang
GitHub
+21
-85
@@ -18,29 +18,19 @@ NpKitEvent** NpKit::cpu_event_buffers_ = nullptr;
|
||||
NpKitEventCollectContext* NpKit::gpu_collect_contexts_ = nullptr;
|
||||
NpKitEventCollectContext* NpKit::cpu_collect_contexts_ = nullptr;
|
||||
|
||||
uint64_t NpKit::base_cpu_timestamp_global_ = 0;
|
||||
uint64_t NpKit::base_cpu_timestamp_local_ = 0;
|
||||
uint64_t NpKit::base_gpu_timestamp_cpu_ = 0;
|
||||
uint64_t NpKit::base_gpu_timestamp_gpu_ = 0;
|
||||
uint64_t* NpKit::cpu_timestamp_ = nullptr;
|
||||
|
||||
__global__ void TimeCalibrationKernel(uint64_t *gpu_timestamp, uint64_t *cpu_timestamp_in, uint64_t *cpu_timestamp_out) {
|
||||
uint64_t gpu_timestamp_1 = NPKIT_GET_GPU_TIMESTAMP();
|
||||
uint64_t cpu_timestamp = *cpu_timestamp_in;
|
||||
uint64_t gpu_timestamp_2 = NPKIT_GET_GPU_TIMESTAMP();
|
||||
*gpu_timestamp = gpu_timestamp_1 + (gpu_timestamp_2 - gpu_timestamp_1) / 2;
|
||||
*cpu_timestamp_out = cpu_timestamp;
|
||||
}
|
||||
|
||||
static volatile bool cpu_timestamp_update_thread_should_stop = false;
|
||||
static volatile uint64_t* volatile_cpu_timestamp = nullptr;
|
||||
std::thread* NpKit::cpu_timestamp_update_thread_ = nullptr;
|
||||
volatile bool NpKit::cpu_timestamp_update_thread_should_stop_ = false;
|
||||
|
||||
void NpKit::CpuTimestampUpdateThread() {
|
||||
uint64_t init_system_clock = std::chrono::system_clock::now().time_since_epoch().count();
|
||||
uint64_t init_steady_clock = std::chrono::steady_clock::now().time_since_epoch().count();
|
||||
uint64_t curr_steady_clock = 0;
|
||||
while (!cpu_timestamp_update_thread_should_stop) {
|
||||
volatile uint64_t* volatile_cpu_timestamp_ = cpu_timestamp_;
|
||||
while (!cpu_timestamp_update_thread_should_stop_) {
|
||||
curr_steady_clock = std::chrono::steady_clock::now().time_since_epoch().count();
|
||||
*volatile_cpu_timestamp = init_system_clock + (curr_steady_clock - init_steady_clock);
|
||||
*volatile_cpu_timestamp_ = init_system_clock + (curr_steady_clock - init_steady_clock);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,40 +57,12 @@ ncclResult_t NpKit::Init(int rank) {
|
||||
cpu_collect_contexts_[i] = ctx;
|
||||
}
|
||||
|
||||
// Calibrate CPU timestamp
|
||||
uint64_t cpu_timestamp_1 = std::chrono::steady_clock::now().time_since_epoch().count();
|
||||
base_cpu_timestamp_global_ = std::chrono::system_clock::now().time_since_epoch().count();
|
||||
uint64_t cpu_timestamp_2 = std::chrono::steady_clock::now().time_since_epoch().count();
|
||||
base_cpu_timestamp_local_ = cpu_timestamp_1 + (cpu_timestamp_2 - cpu_timestamp_1) / 2;
|
||||
|
||||
// Calibrate GPU timestamp
|
||||
uint64_t *cpu_timestamp_in = nullptr;
|
||||
uint64_t *cpu_timestamp_out = nullptr;
|
||||
uint64_t *gpu_timestamp = nullptr;
|
||||
NCCLCHECK(ncclCudaHostCalloc(&cpu_timestamp_in, 1));
|
||||
NCCLCHECK(ncclCudaCalloc(&cpu_timestamp_out, 1));
|
||||
NCCLCHECK(ncclCudaCalloc(&gpu_timestamp, 1));
|
||||
|
||||
cpu_timestamp_update_thread_should_stop = false;
|
||||
volatile_cpu_timestamp = cpu_timestamp_in;
|
||||
*volatile_cpu_timestamp = 0;
|
||||
|
||||
hipStream_t timingStream;
|
||||
CUDACHECK(hipStreamCreateWithFlags(&timingStream, hipStreamNonBlocking));
|
||||
|
||||
std::thread cpu_timestamp_update_thread(CpuTimestampUpdateThread);
|
||||
while (!*volatile_cpu_timestamp);
|
||||
TimeCalibrationKernel<<<1, 1, 0, timingStream>>>(gpu_timestamp, cpu_timestamp_in, cpu_timestamp_out);
|
||||
CUDACHECK(hipStreamSynchronize(timingStream));
|
||||
cpu_timestamp_update_thread_should_stop = true;
|
||||
cpu_timestamp_update_thread.join();
|
||||
|
||||
NCCLCHECK(ncclCudaMemcpy(&base_gpu_timestamp_cpu_, cpu_timestamp_out, 1));
|
||||
NCCLCHECK(ncclCudaMemcpy(&base_gpu_timestamp_gpu_, gpu_timestamp, 1));
|
||||
|
||||
NCCLCHECK(ncclCudaHostFree(cpu_timestamp_in));
|
||||
CUDACHECK(hipFree(cpu_timestamp_out));
|
||||
CUDACHECK(hipFree(gpu_timestamp));
|
||||
// Init timestamp
|
||||
NCCLCHECK(ncclCudaHostCalloc(&cpu_timestamp_, 1));
|
||||
volatile uint64_t* volatile_cpu_timestamp = cpu_timestamp_;
|
||||
*volatile_cpu_timestamp = std::chrono::system_clock::now().time_since_epoch().count();
|
||||
cpu_timestamp_update_thread_should_stop_ = false;
|
||||
cpu_timestamp_update_thread_ = new std::thread(CpuTimestampUpdateThread);
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -164,45 +126,16 @@ ncclResult_t NpKit::Dump(const std::string& dump_dir) {
|
||||
gpu_clock_rate_file.write(clock_rate_str.c_str(), clock_rate_str.length());
|
||||
gpu_clock_rate_file.close();
|
||||
|
||||
// Dump clock calibration info
|
||||
dump_file_path = dump_dir;
|
||||
dump_file_path += "/clock_calibration_cpu_global_rank_";
|
||||
dump_file_path += std::to_string(rank_);
|
||||
std::string base_cpu_timestamp_global_str = std::to_string(base_cpu_timestamp_global_);
|
||||
auto base_cpu_timestamp_global_file = std::fstream(dump_file_path, std::ios::out);
|
||||
base_cpu_timestamp_global_file.write(base_cpu_timestamp_global_str.c_str(), base_cpu_timestamp_global_str.length());
|
||||
base_cpu_timestamp_global_file.close();
|
||||
|
||||
dump_file_path = dump_dir;
|
||||
dump_file_path += "/clock_calibration_cpu_local_rank_";
|
||||
dump_file_path += std::to_string(rank_);
|
||||
std::string base_cpu_timestamp_local_str = std::to_string(base_cpu_timestamp_local_);
|
||||
auto base_cpu_timestamp_local_file = std::fstream(dump_file_path, std::ios::out);
|
||||
base_cpu_timestamp_local_file.write(base_cpu_timestamp_local_str.c_str(), base_cpu_timestamp_local_str.length());
|
||||
base_cpu_timestamp_local_file.close();
|
||||
|
||||
dump_file_path = dump_dir;
|
||||
dump_file_path += "/clock_calibration_gpu_cpu_rank_";
|
||||
dump_file_path += std::to_string(rank_);
|
||||
std::string base_gpu_timestamp_cpu_str = std::to_string(base_gpu_timestamp_cpu_);
|
||||
auto base_gpu_timestamp_cpu_file = std::fstream(dump_file_path, std::ios::out);
|
||||
base_gpu_timestamp_cpu_file.write(base_gpu_timestamp_cpu_str.c_str(), base_gpu_timestamp_cpu_str.length());
|
||||
base_gpu_timestamp_cpu_file.close();
|
||||
|
||||
dump_file_path = dump_dir;
|
||||
dump_file_path += "/clock_calibration_gpu_gpu_rank_";
|
||||
dump_file_path += std::to_string(rank_);
|
||||
std::string base_gpu_timestamp_gpu_str = std::to_string(base_gpu_timestamp_gpu_);
|
||||
auto base_gpu_timestamp_gpu_file = std::fstream(dump_file_path, std::ios::out);
|
||||
base_gpu_timestamp_gpu_file.write(base_gpu_timestamp_gpu_str.c_str(), base_gpu_timestamp_gpu_str.length());
|
||||
base_gpu_timestamp_gpu_file.close();
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t NpKit::Shutdown() {
|
||||
uint64_t i = 0;
|
||||
|
||||
// Stop CPU timestamp updating thread
|
||||
cpu_timestamp_update_thread_should_stop_ = true;
|
||||
cpu_timestamp_update_thread_->join();
|
||||
|
||||
// Free CPU event data structures
|
||||
for (i = 0; i < kNumCpuEventBuffers; i++) {
|
||||
free(cpu_event_buffers_[i]);
|
||||
@@ -217,6 +150,9 @@ ncclResult_t NpKit::Shutdown() {
|
||||
free(gpu_event_buffers_);
|
||||
CUDACHECK(hipFree(gpu_collect_contexts_));
|
||||
|
||||
// Free timestamp
|
||||
NCCLCHECK(ncclCudaHostFree(cpu_timestamp_));
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -236,6 +172,6 @@ void NpKit::CollectCpuEvent(uint8_t type, uint32_t size, uint32_t rsvd, uint64_t
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t NpKit::GetCpuTimestamp() {
|
||||
return std::chrono::steady_clock::now().time_since_epoch().count();
|
||||
uint64_t* NpKit::GetCpuTimestamp() {
|
||||
return cpu_timestamp_;
|
||||
}
|
||||
|
||||
مرجع در شماره جدید
Block a user