You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/01/02 18:13:36 UTC
[GitHub] piiswrong closed pull request #9219: Add storage fallback msg to copyfromto
piiswrong closed pull request #9219: Add storage fallback msg to copyfromto
URL: https://github.com/apache/incubator-mxnet/pull/9219
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/src/common/utils.h b/src/common/utils.h
index 038ab2a047..90159cbf34 100644
--- a/src/common/utils.h
+++ b/src/common/utils.h
@@ -339,6 +339,32 @@ inline std::string stype_string(const int x) {
return "unknown";
}
+/*! \brief get string representation of device type */
+inline std::string dev_type_string(const int dev_type) {
+ switch (dev_type) {
+ case Context::kCPU:
+ return "cpu";
+ case Context::kGPU:
+ return "gpu";
+ case Context::kCPUPinned:
+ return "cpu_pinned";
+ case Context::kCPUShared:
+ return "cpu_shared";
+ }
+ return "unknown";
+}
+
+/*! \brief log message once. Intended for storage fallback warning messages. */
+inline void LogOnce(const std::string& message) {
+ typedef dmlc::ThreadLocalStore<std::unordered_set<std::string>> LogStore;
+ auto log_store = LogStore::Get();
+ if (log_store->find(message) == log_store->end()) {
+ LOG(INFO) << message;
+ log_store->insert(message);
+ }
+}
+
+
// heuristic to dermine number of threads per GPU
inline int GetNumThreadPerGPU() {
// This is resource efficient option.
diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc
index f09f168977..212fd7ce48 100644
--- a/src/ndarray/ndarray.cc
+++ b/src/ndarray/ndarray.cc
@@ -531,11 +531,28 @@ void CopyFromTo(const NDArray& from, const NDArray& to, int priority) {
std::vector<Engine::VarHandle> mutable_vars(1, to.var());
std::vector<Resource> requested;
- if (a == gpu::kDevMask && from_stype != to_stype) {
- Resource rsc = ResourceManager::Get()->Request(from_ctx,
- ResourceRequest(ResourceRequest::kTempSpace));
- requested.push_back(rsc);
- mutable_vars.push_back(rsc.var);
+ if (from_stype != to_stype) {
+ using namespace common;
+ static bool log = dmlc::GetEnv("MXNET_STORAGE_FALLBACK_LOG_VERBOSE", true);
+ if (log) {
+ std::ostringstream os;
+ os << "\nStorage fallback detected:\n"
+ << "Copy from " << stype_string(from_stype) << " storage type on " << dev_type_string(a)
+ << " to " << stype_string(to_stype) << " storage type on " << dev_type_string(b)
+ << ".\nA temporary ndarray with " << stype_string(to_stype)
+ << " storage type will be generated in order to perform the copy. "
+ << "You can set environment variable "
+ << "MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.";
+ LogOnce(os.str());
+ }
+
+ // request temp resource if cast_storage performs on GPU
+ if (a == gpu::kDevMask) {
+ Resource rsc = ResourceManager::Get()->Request(from_ctx,
+ ResourceRequest(ResourceRequest::kTempSpace));
+ requested.push_back(rsc);
+ mutable_vars.push_back(rsc.var);
+ }
}
if (a == cpu::kDevMask && b == cpu::kDevMask) {
diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h
index 560d11e67e..329db02d81 100644
--- a/src/operator/operator_common.h
+++ b/src/operator/operator_common.h
@@ -223,7 +223,7 @@ inline bool dispatch_mode_assign(DispatchMode *y, const DispatchMode& x) {
{ \
if (!shape_assign(&(shape_array)[index], TShape(shape))) { \
std::ostringstream os; \
- os << "Shape inconsistent, Provided=" << (shape_array)[index] << ',' \
+ os << "Shape inconsistent, Provided = " << (shape_array)[index] << ','\
<< " inferred shape=" << shape; \
throw ::mxnet::op::InferShapeError(os.str(), index); \
} \
@@ -240,9 +240,9 @@ inline bool dispatch_mode_assign(DispatchMode *y, const DispatchMode& x) {
{ \
if (!type_assign(&(type_array)[index], type)) { \
std::ostringstream os; \
- os << "Type inconsistent, Provided=" \
+ os << "Type inconsistent, Provided = " \
<< type_string((type_array)[index]) << ',' \
- << " inferred type=" << type_string(type); \
+ << " inferred type = " << type_string(type); \
throw ::mxnet::op::InferTypeError(os.str(), index); \
} \
}
@@ -258,9 +258,9 @@ inline bool dispatch_mode_assign(DispatchMode *y, const DispatchMode& x) {
{ \
if (!type_assign(&(type_array)[index], type)) { \
std::ostringstream os; \
- os << "Storage type inconsistent, Provided=" \
+ os << "Storage type inconsistent, Provided = " \
<< common::stype_string((type_array)[index]) << ',' \
- << " inferred storage type=" << common::stype_string(type); \
+ << " inferred storage type = " << common::stype_string(type); \
throw ::mxnet::op::InferStorageTypeError(os.str(), index); \
} \
}
@@ -274,11 +274,11 @@ inline bool dispatch_mode_assign(DispatchMode *y, const DispatchMode& x) {
*/
#define DISPATCH_MODE_ASSIGN_CHECK(type_array, index, type) \
{ \
- if (!dispatch_mode_assign(&(type_array)[index], type)) { \
+ if (!dispatch_mode_assign(&(type_array)[index], type)) { \
std::ostringstream os; \
- os << "Dispatch mode inconsistent, Provided=" \
+ os << "Dispatch mode inconsistent, Provided = " \
<< common::dispatch_mode_string((type_array)[index]) << ',' \
- << " inferred mode=" << common::dispatch_mode_string(type); \
+ << " inferred mode = " << common::dispatch_mode_string(type); \
throw ::mxnet::op::InferStorageTypeError(os.str(), index); \
} \
}
@@ -501,7 +501,7 @@ inline std::string operator_stype_string(const nnvm::NodeAttrs& attrs,
result += "\"" + kv.first + "\" : " + kv.second + ", ";
}
result += "}\n";
- result += "context.dev_mask = " + std::to_string(dev_mask);
+ result += "context.dev_mask = " + common::dev_type_string(dev_mask);
return result;
}
@@ -527,22 +527,18 @@ inline void LogStorageFallback(const nnvm::NodeAttrs& attrs,
const int dev_mask,
const std::vector<int>* in_attrs,
const std::vector<int>* out_attrs) {
- using namespace op;
- auto warning_printed = dmlc::ThreadLocalStore<std::unordered_set<std::string>>::Get();
- static bool log_verbose = dmlc::GetEnv("MXNET_STORAGE_FALLBACK_LOG_VERBOSE", true);
- if (log_verbose) {
- std::string warning = operator_stype_string(attrs, dev_mask, *in_attrs, *out_attrs);
- if (warning_printed->find(warning) == warning_printed->end()) {
- LOG(INFO) << "\nStorage fallback detected:\n" << warning
- << "\nThe operator with default storage type will be dispatched for execution. "
- << "You're seeing this warning message because the operator above is unable to "
- << "process the given ndarrays with specified storage types and parameter. "
- << "Temporary dense ndarrays are generated in order to execute the operator. "
- << "You can set environment variable MXNET_STORAGE_FALLBACK_LOG_VERBOSE "
- << "to 0 to suppress the warnings.";
- warning_printed->insert(warning);
- }
- }
+ static bool log = dmlc::GetEnv("MXNET_STORAGE_FALLBACK_LOG_VERBOSE", true);
+ if (!log) return;
+ const std::string op_str = op::operator_stype_string(attrs, dev_mask, *in_attrs, *out_attrs);
+ std::ostringstream os;
+ os << "\nStorage type fallback detected:\n" << op_str
+ << "\nThe operator with default storage type will be dispatched for execution. "
+ << "You're seeing this warning message because the operator above is unable to "
+ << "process the given ndarrays with specified storage types, context and parameter. "
+ << "Temporary dense ndarrays are generated in order to execute the operator. "
+ << "You can set environment variable "
+ << "MXNET_STORAGE_FALLBACK_LOG_VERBOSE to 0 to suppress this warning.";
+ common::LogOnce(os.str());
}
} // namespace op
diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py
index e3d977df65..40a922cf5c 100644
--- a/tests/python/unittest/test_executor.py
+++ b/tests/python/unittest/test_executor.py
@@ -77,40 +77,44 @@ def check_bind_with_uniform(uf, gf, dim, sf=None, lshape=None, rshape=None):
assert reldiff(rhs_grad.asnumpy(), rhs_grad2) < 1e-6
-def test_bind(disable_bulk_exec=False):
- if disable_bulk_exec:
- prev_bulk_inf_val = mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", "0", "1")
- prev_bulk_train_val = mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", "0", "1")
-
- np.random.seed(0)
- nrepeat = 10
- maxdim = 4
- for repeat in range(nrepeat):
- for dim in range(1, maxdim):
- check_bind_with_uniform(lambda x, y: x + y,
- lambda g, x, y: (g, g),
- dim)
- check_bind_with_uniform(lambda x, y: x - y,
- lambda g, x, y: (g, -g),
- dim)
- check_bind_with_uniform(lambda x, y: x * y,
- lambda g, x, y: (y * g, x * g),
- dim)
- check_bind_with_uniform(lambda x, y: x / y,
- lambda g, x, y: (g / y, -x * g/ (y**2)),
- dim)
-
- check_bind_with_uniform(lambda x, y: np.maximum(x, y),
- lambda g, x, y: (g * (x>y), g * (y>x)),
- dim,
- sf=mx.symbol.maximum)
- check_bind_with_uniform(lambda x, y: np.minimum(x, y),
- lambda g, x, y: (g * (x<y), g * (y<x)),
- dim,
- sf=mx.symbol.minimum)
- if disable_bulk_exec:
- mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", prev_bulk_inf_val)
- mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", prev_bulk_train_val)
+def test_bind():
+ def check_bind(disable_bulk_exec):
+ if disable_bulk_exec:
+ prev_bulk_inf_val = mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", "0", "1")
+ prev_bulk_train_val = mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", "0", "1")
+
+ np.random.seed(0)
+ nrepeat = 10
+ maxdim = 4
+ for repeat in range(nrepeat):
+ for dim in range(1, maxdim):
+ check_bind_with_uniform(lambda x, y: x + y,
+ lambda g, x, y: (g, g),
+ dim)
+ check_bind_with_uniform(lambda x, y: x - y,
+ lambda g, x, y: (g, -g),
+ dim)
+ check_bind_with_uniform(lambda x, y: x * y,
+ lambda g, x, y: (y * g, x * g),
+ dim)
+ check_bind_with_uniform(lambda x, y: x / y,
+ lambda g, x, y: (g / y, -x * g/ (y**2)),
+ dim)
+
+ check_bind_with_uniform(lambda x, y: np.maximum(x, y),
+ lambda g, x, y: (g * (x>y), g * (y>x)),
+ dim,
+ sf=mx.symbol.maximum)
+ check_bind_with_uniform(lambda x, y: np.minimum(x, y),
+ lambda g, x, y: (g * (x<y), g * (y<x)),
+ dim,
+ sf=mx.symbol.minimum)
+ if disable_bulk_exec:
+ mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_INFERENCE", prev_bulk_inf_val)
+ mx.test_utils.set_env_var("MXNET_EXEC_BULK_EXEC_TRAIN", prev_bulk_train_val)
+
+ check_bind(True)
+ check_bind(False)
def test_dot():
np.random.seed(0)
@@ -154,6 +158,5 @@ def test_reshape():
assert np.all(exe.outputs[0].asnumpy() == 4)
if __name__ == "__main__":
- test_bind(disable_bulk_exec=False)
- test_bind(disable_bulk_exec=True)
- test_reshape()
+ import nose
+ nose.runmodule()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services