[GitHub] piiswrong closed pull request #8838: fix #8262

[GitBox <gi...@apache.org>]

piiswrong closed pull request #8838: fix #8262
URL: https://github.com/apache/incubator-mxnet/pull/8838
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/src/kvstore/comm.h b/src/kvstore/comm.h
index 5e15c2a085..5429df70b1 100644
--- a/src/kvstore/comm.h
+++ b/src/kvstore/comm.h
@@ -251,6 +251,8 @@ class CommCPU : public Comm {
             [=](RunContext rctx, Engine::CallbackOnComplete on_complete) {
               CopyRetainedRowsToGPU(rctx.get_stream<cpu>(), rctx.get_stream<gpu>(),
                                     src, row_id, out);
+              // wait for GPU operations to complete
+              rctx.get_stream<gpu>()->Wait();
               on_complete();
             }, out->ctx(), {src.var(), row_id.var()}, {out->var()},
             FnProperty::kCopyToGPU, priority, PROFILER_MESSAGE("KVStoreCopyRetainedRowsToGPU"));
diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index 4a096d1bd0..25fd50503a 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -505,10 +505,6 @@ void CopyFromToImpl(const NDArray& from, const NDArray& to, RunContext rctx) {
       LOG(FATAL) << "unknown storage type" << to_stype;
     }
   }
-  if (is_same<from_xpu, mshadow::gpu>::value || is_same<to_xpu, mshadow::gpu>::value) {
-    // Wait GPU kernel to complete
-    rctx.get_stream<gpu>()->Wait();
-  }
 }
 
 void CopyFromTo(const NDArray& from, const NDArray& to, int priority) {
@@ -540,6 +536,7 @@ void CopyFromTo(const NDArray& from, const NDArray& to, int priority) {
       Engine::Get()->PushAsync(
         [from, to](RunContext ctx, Engine::CallbackOnComplete on_complete) {
           CopyFromToImpl<cpu, gpu>(from, to, ctx);
+          ctx.get_stream<gpu>()->Wait();
           on_complete();
         }, to.ctx(), const_vars, {to.var()},
         FnProperty::kCopyToGPU, priority, PROFILER_MESSAGE("CopyCPU2GPU"));
@@ -547,6 +544,7 @@ void CopyFromTo(const NDArray& from, const NDArray& to, int priority) {
       Engine::Get()->PushAsync(
         [from, to](RunContext ctx, Engine::CallbackOnComplete on_complete) {
           CopyFromToImpl<gpu, cpu>(from, to, ctx);
+          ctx.get_stream<gpu>()->Wait();
           on_complete();
         }, from.ctx(), const_vars, {to.var()},
         FnProperty::kCopyFromGPU, priority, PROFILER_MESSAGE("CopyGPU2CPU"));
@@ -554,6 +552,7 @@ void CopyFromTo(const NDArray& from, const NDArray& to, int priority) {
       Engine::Get()->PushAsync(
         [from, to](RunContext ctx, Engine::CallbackOnComplete on_complete) {
           CopyFromToImpl<gpu, gpu>(from, to, ctx);
+          ctx.get_stream<gpu>()->Wait();
           on_complete();
         }, from.ctx(), const_vars, {to.var()},
         from.dtype() != to.dtype() ? FnProperty::kNormal : FnProperty::kCopyFromGPU,
diff --git a/tests/python/gpu/test_kvstore_gpu.py b/tests/python/gpu/test_kvstore_gpu.py
index 517d2e7ca4..20528be664 100644
--- a/tests/python/gpu/test_kvstore_gpu.py
+++ b/tests/python/gpu/test_kvstore_gpu.py
@@ -36,7 +36,6 @@ def init_kv_with_str(stype='default'):
     return kv
 
 
-@unittest.skip("Test fails intermittently. Temporarily disabled until fixed. Tracked at https://github.com/apache/incubator-mxnet/issues/8262")
 def test_row_sparse_pull():
     kv = init_kv_with_str('row_sparse')
     kv.init('e', mx.nd.ones(shape).tostype('row_sparse'))


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services