You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by cs...@apache.org on 2024/01/22 20:33:49 UTC

(tvm) branch main updated: [Disco][3rdparty] Add latency optimized all reduce kernels.

This is an automated email from the ASF dual-hosted git repository.

csullivan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git


The following commit(s) were added to refs/heads/main by this push:
     new ee5c994a59 [Disco][3rdparty] Add latency optimized all reduce kernels.
ee5c994a59 is described below

commit ee5c994a591f58755e69d51bfaabe075df826af4
Author: Chris Sullivan <cs...@octoml.ai>
AuthorDate: Mon Jan 22 14:44:15 2024 +0000

    [Disco][3rdparty] Add latency optimized all reduce kernels.
---
 .gitmodules                    |  3 +++
 3rdparty/trt-llm-allreduce     |  1 +
 CMakeLists.txt                 |  3 +++
 cmake/modules/CUDA.cmake       |  4 ++++
 src/runtime/disco/nccl/nccl.cc | 12 ++++++++++++
 5 files changed, 23 insertions(+)

diff --git a/.gitmodules b/.gitmodules
index b5102d9a9b..fd9363b144 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -31,3 +31,6 @@
 [submodule "3rdparty/flashinfer"]
 	path = 3rdparty/flashinfer
 	url = https://github.com/flashinfer-ai/flashinfer.git
+[submodule "3rdparty/trt-llm-allreduce"]
+	path = 3rdparty/trt-llm-allreduce
+	url = git@github.com:csullivan/trt-llm-allreduce.git
diff --git a/3rdparty/trt-llm-allreduce b/3rdparty/trt-llm-allreduce
new file mode 160000
index 0000000000..84d707ae96
--- /dev/null
+++ b/3rdparty/trt-llm-allreduce
@@ -0,0 +1 @@
+Subproject commit 84d707ae96651fcbb872edda9b5c7e2897f81bf5
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 058f477dbd..4a2ccae919 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -908,6 +908,9 @@ if(USE_CUDA AND USE_NCCL)
   find_library(LIBRT rt)
   target_link_libraries(tvm PRIVATE nccl ${LIBRT})
   target_link_libraries(tvm_runtime PRIVATE nccl ${LIBRT})
+  install(TARGETS trtllm_allreduce EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
+  target_link_libraries(tvm PRIVATE -Wl,--no-as-needed trtllm_allreduce)
+  target_link_libraries(tvm_runtime PRIVATE -Wl,--no-as-needed trtllm_allreduce)
 endif()
 
 if(USE_ROCM AND USE_RCCL)
diff --git a/cmake/modules/CUDA.cmake b/cmake/modules/CUDA.cmake
index 84f466f591..e2c55d0c68 100644
--- a/cmake/modules/CUDA.cmake
+++ b/cmake/modules/CUDA.cmake
@@ -47,6 +47,10 @@ if(USE_CUDA)
     set(CMAKE_CUDA_ARCHITECTURES native)
   endif()
 
+  if(USE_CUDA AND USE_NCCL)
+    add_subdirectory(${PROJECT_SOURCE_DIR}/3rdparty/trt-llm-allreduce)
+  endif()
+
   if(USE_CUDNN)
     message(STATUS "Build with cuDNN support")
     include_directories(SYSTEM ${CUDA_CUDNN_INCLUDE_DIRS})
diff --git a/src/runtime/disco/nccl/nccl.cc b/src/runtime/disco/nccl/nccl.cc
index 61c307c673..4be8133229 100644
--- a/src/runtime/disco/nccl/nccl.cc
+++ b/src/runtime/disco/nccl/nccl.cc
@@ -24,6 +24,7 @@
 #include <tvm/runtime/registry.h>
 
 #include <cstring>
+#include <memory>
 #include <mutex>
 #include <sstream>
 #include <vector>
@@ -38,6 +39,7 @@
 #if TVM_NCCL_RCCL_SWITCH == 0
 #include <nccl.h>
 
+#include "../../../../3rdparty/trt-llm-allreduce/include/cuda_allreduce.h"
 #include "../../cuda/cuda_common.h"
 #else
 #include <rccl/rccl.h>
@@ -140,6 +142,7 @@ struct CCLThreadLocalContext {
   int device_id;
   deviceStream_t default_stream = nullptr;
   ncclComm_t comm;
+  std::unique_ptr<CustomAllReduce> custom_allreduce;
 
   void Clear() {
     NCCL_CALL(ncclCommDestroy(comm));
@@ -190,6 +193,8 @@ void InitCCLPerWorker(IntTuple device_ids, std::string unique_id_bytes) {
   worker->ccl = TVM_DISCO_CCL_NAME;
   ctx->worker = worker;
   ctx->device_id = device_id;
+  ctx->custom_allreduce =
+      std::make_unique<CustomAllReduce>(worker->num_workers, worker->worker_id, ctx->comm);
   // Initialize the communicator
   ncclUniqueId id;
   std::memcpy(id.internal, unique_id_bytes.data(), NCCL_UNIQUE_ID_BYTES);
@@ -201,6 +206,13 @@ void AllReduce(NDArray send, ReduceKind reduce_kind, NDArray recv) {
   ShapeTuple shape = send.Shape();
   int64_t numel = shape->Product();
   deviceStream_t stream = ctx->GetDefaultStream();
+  // TODO(csullivan) make this work
+  // 1. pass type in
+  // 2. src and dest args
+  // 3. some strategy selection outside, if (!enqueu) do nccl?
+  // 3. reduce kind
+  // 4. pass stream in to custom api
+  // ctx->custom_allreduce->enqueue(send->data, numel);
   NCCL_CALL(ncclAllReduce(send->data, recv->data, numel,
                           /*datatype=*/AsNCCLDataType(DataType(send->dtype)),
                           /*op=*/AsNCCLRedOp(reduce_kind), ctx->comm, stream));