You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by cs...@apache.org on 2024/01/22 20:33:49 UTC
(tvm) branch main updated: [Disco][3rdparty] Add latency optimized all reduce kernels.
This is an automated email from the ASF dual-hosted git repository.
csullivan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new ee5c994a59 [Disco][3rdparty] Add latency optimized all reduce kernels.
ee5c994a59 is described below
commit ee5c994a591f58755e69d51bfaabe075df826af4
Author: Chris Sullivan <cs...@octoml.ai>
AuthorDate: Mon Jan 22 14:44:15 2024 +0000
[Disco][3rdparty] Add latency optimized all reduce kernels.
---
.gitmodules | 3 +++
3rdparty/trt-llm-allreduce | 1 +
CMakeLists.txt | 3 +++
cmake/modules/CUDA.cmake | 4 ++++
src/runtime/disco/nccl/nccl.cc | 12 ++++++++++++
5 files changed, 23 insertions(+)
diff --git a/.gitmodules b/.gitmodules
index b5102d9a9b..fd9363b144 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -31,3 +31,6 @@
[submodule "3rdparty/flashinfer"]
path = 3rdparty/flashinfer
url = https://github.com/flashinfer-ai/flashinfer.git
+[submodule "3rdparty/trt-llm-allreduce"]
+ path = 3rdparty/trt-llm-allreduce
+ url = git@github.com:csullivan/trt-llm-allreduce.git
diff --git a/3rdparty/trt-llm-allreduce b/3rdparty/trt-llm-allreduce
new file mode 160000
index 0000000000..84d707ae96
--- /dev/null
+++ b/3rdparty/trt-llm-allreduce
@@ -0,0 +1 @@
+Subproject commit 84d707ae96651fcbb872edda9b5c7e2897f81bf5
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 058f477dbd..4a2ccae919 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -908,6 +908,9 @@ if(USE_CUDA AND USE_NCCL)
find_library(LIBRT rt)
target_link_libraries(tvm PRIVATE nccl ${LIBRT})
target_link_libraries(tvm_runtime PRIVATE nccl ${LIBRT})
+ install(TARGETS trtllm_allreduce EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
+ target_link_libraries(tvm PRIVATE -Wl,--no-as-needed trtllm_allreduce)
+ target_link_libraries(tvm_runtime PRIVATE -Wl,--no-as-needed trtllm_allreduce)
endif()
if(USE_ROCM AND USE_RCCL)
diff --git a/cmake/modules/CUDA.cmake b/cmake/modules/CUDA.cmake
index 84f466f591..e2c55d0c68 100644
--- a/cmake/modules/CUDA.cmake
+++ b/cmake/modules/CUDA.cmake
@@ -47,6 +47,10 @@ if(USE_CUDA)
set(CMAKE_CUDA_ARCHITECTURES native)
endif()
+ if(USE_CUDA AND USE_NCCL)
+ add_subdirectory(${PROJECT_SOURCE_DIR}/3rdparty/trt-llm-allreduce)
+ endif()
+
if(USE_CUDNN)
message(STATUS "Build with cuDNN support")
include_directories(SYSTEM ${CUDA_CUDNN_INCLUDE_DIRS})
diff --git a/src/runtime/disco/nccl/nccl.cc b/src/runtime/disco/nccl/nccl.cc
index 61c307c673..4be8133229 100644
--- a/src/runtime/disco/nccl/nccl.cc
+++ b/src/runtime/disco/nccl/nccl.cc
@@ -24,6 +24,7 @@
#include <tvm/runtime/registry.h>
#include <cstring>
+#include <memory>
#include <mutex>
#include <sstream>
#include <vector>
@@ -38,6 +39,7 @@
#if TVM_NCCL_RCCL_SWITCH == 0
#include <nccl.h>
+#include "../../../../3rdparty/trt-llm-allreduce/include/cuda_allreduce.h"
#include "../../cuda/cuda_common.h"
#else
#include <rccl/rccl.h>
@@ -140,6 +142,7 @@ struct CCLThreadLocalContext {
int device_id;
deviceStream_t default_stream = nullptr;
ncclComm_t comm;
+ std::unique_ptr<CustomAllReduce> custom_allreduce;
void Clear() {
NCCL_CALL(ncclCommDestroy(comm));
@@ -190,6 +193,8 @@ void InitCCLPerWorker(IntTuple device_ids, std::string unique_id_bytes) {
worker->ccl = TVM_DISCO_CCL_NAME;
ctx->worker = worker;
ctx->device_id = device_id;
+ ctx->custom_allreduce =
+ std::make_unique<CustomAllReduce>(worker->num_workers, worker->worker_id, ctx->comm);
// Initialize the communicator
ncclUniqueId id;
std::memcpy(id.internal, unique_id_bytes.data(), NCCL_UNIQUE_ID_BYTES);
@@ -201,6 +206,13 @@ void AllReduce(NDArray send, ReduceKind reduce_kind, NDArray recv) {
ShapeTuple shape = send.Shape();
int64_t numel = shape->Product();
deviceStream_t stream = ctx->GetDefaultStream();
+ // TODO(csullivan) make this work
+ // 1. pass type in
+ // 2. src and dest args
+ // 3. some strategy selection outside, if (!enqueu) do nccl?
+ // 3. reduce kind
+ // 4. pass stream in to custom api
+ // ctx->custom_allreduce->enqueue(send->data, numel);
NCCL_CALL(ncclAllReduce(send->data, recv->data, numel,
/*datatype=*/AsNCCLDataType(DataType(send->dtype)),
/*op=*/AsNCCLRedOp(reduce_kind), ctx->comm, stream));