You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/09/05 00:57:57 UTC
[GitHub] zheng-da closed pull request #12456: [WIP] Multithreading inference.
zheng-da closed pull request #12456: [WIP] Multithreading inference.
URL: https://github.com/apache/incubator-mxnet/pull/12456
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/example/image-classification/predict-cpp/Makefile b/example/image-classification/predict-cpp/Makefile
index e0c0bc65729..5c084119b96 100644
--- a/example/image-classification/predict-cpp/Makefile
+++ b/example/image-classification/predict-cpp/Makefile
@@ -15,7 +15,7 @@ LDFLAGS+=`pkg-config --libs opencv`
export MXNET_ROOT=`pwd`/../../..
CFLAGS+=-Wall -I$(MXNET_ROOT)/include
-LDFLAGS+=$(MXNET_ROOT)/lib/libmxnet.so
+LDFLAGS+=$(MXNET_ROOT)/lib/libmxnet.so -lpthread
image-classification-predict: image-classification-predict.o
g++ -O3 -o image-classification-predict image-classification-predict.o $(LDFLAGS)
diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc
index 186107bd530..4c6b1e46cb7 100644
--- a/example/image-classification/predict-cpp/image-classification-predict.cc
+++ b/example/image-classification/predict-cpp/image-classification-predict.cc
@@ -37,6 +37,7 @@
#include <fstream>
#include <vector>
#include <memory>
+#include <thread>
#include <iomanip>
#include <opencv2/opencv.hpp>
// Path for c_predict_api
@@ -179,6 +180,46 @@ void PrintOutputResult(const std::vector<float>& data, const std::vector<std::st
"accuracy=" << std::setprecision(8) << best_accuracy << ")" << std::endl;
}
+void predict(PredictorHandle pred_hnd, const std::vector<mx_float> &image_data,
+ NDListHandle nd_hnd, int i) {
+ std::string synset_file = "synset.txt";
+ auto image_size = image_data.size();
+ // Set Input Image
+ MXPredSetInput(pred_hnd, "data", image_data.data(), static_cast<mx_uint>(image_size));
+
+ // Do Predict Forward
+ MXPredForward(pred_hnd);
+
+ mx_uint output_index = 0;
+
+ mx_uint* shape = nullptr;
+ mx_uint shape_len;
+
+ // Get Output Result
+ MXPredGetOutputShape(pred_hnd, output_index, &shape, &shape_len);
+
+ std::size_t size = 1;
+ for (mx_uint i = 0; i < shape_len; ++i) { size *= shape[i]; }
+
+ std::vector<float> data(size);
+
+ MXPredGetOutput(pred_hnd, output_index, &(data[0]), static_cast<mx_uint>(size));
+
+ // Release NDList
+ if (nd_hnd) {
+ MXNDListFree(nd_hnd);
+ }
+
+ // Release Predictor
+ MXPredFree(pred_hnd);
+
+ // Synset path for your model, you have to modify it
+ auto synset = LoadSynset(synset_file);
+
+ // Print Output Data
+ PrintOutputResult(data, synset);
+}
+
int main(int argc, char* argv[]) {
if (argc < 2) {
std::cout << "No test image here." << std::endl
@@ -189,10 +230,10 @@ int main(int argc, char* argv[]) {
std::string test_file(argv[1]);
// Models path for your model, you have to modify it
- std::string json_file = "model/Inception/Inception-BN-symbol.json";
- std::string param_file = "model/Inception/Inception-BN-0126.params";
- std::string synset_file = "model/Inception/synset.txt";
- std::string nd_file = "model/Inception/mean_224.nd";
+ std::string json_file = "Inception-BN-symbol.json";
+ std::string param_file = "Inception-BN-0126.params";
+ std::string synset_file = "synset.txt";
+ std::string nd_file = "mean_224.nd";
BufferFile json_data(json_file);
BufferFile param_data(param_file);
@@ -214,14 +255,14 @@ int main(int argc, char* argv[]) {
static_cast<mx_uint>(channels),
static_cast<mx_uint>(height),
static_cast<mx_uint>(width) };
- PredictorHandle pred_hnd = nullptr;
+ std::vector<PredictorHandle> pred_hnds(8, nullptr);
if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
return EXIT_FAILURE;
}
// Create Predictor
- MXPredCreate(static_cast<const char*>(json_data.GetBuffer()),
+ MXPredCreateMultithread(static_cast<const char*>(json_data.GetBuffer()),
static_cast<const char*>(param_data.GetBuffer()),
static_cast<int>(param_data.GetLength()),
dev_type,
@@ -230,8 +271,10 @@ int main(int argc, char* argv[]) {
input_keys,
input_shape_indptr,
input_shape_data,
- &pred_hnd);
- assert(pred_hnd);
+ pred_hnds.size(),
+ pred_hnds.data());
+ for (auto hnd : pred_hnds)
+ assert(hnd);
auto image_size = static_cast<std::size_t>(width * height * channels);
@@ -259,40 +302,22 @@ int main(int argc, char* argv[]) {
GetImageFile(test_file, image_data.data(), channels, cv::Size(width, height), nd_data);
- // Set Input Image
- MXPredSetInput(pred_hnd, "data", image_data.data(), static_cast<mx_uint>(image_size));
-
- // Do Predict Forward
- MXPredForward(pred_hnd);
-
- mx_uint output_index = 0;
-
- mx_uint* shape = nullptr;
- mx_uint shape_len;
-
- // Get Output Result
- MXPredGetOutputShape(pred_hnd, output_index, &shape, &shape_len);
-
- std::size_t size = 1;
- for (mx_uint i = 0; i < shape_len; ++i) { size *= shape[i]; }
-
- std::vector<float> data(size);
-
- MXPredGetOutput(pred_hnd, output_index, &(data[0]), static_cast<mx_uint>(size));
-
- // Release NDList
- if (nd_hnd) {
- MXNDListFree(nd_hnd);
- }
-
- // Release Predictor
- MXPredFree(pred_hnd);
-
- // Synset path for your model, you have to modify it
- auto synset = LoadSynset(synset_file);
-
- // Print Output Data
- PrintOutputResult(data, synset);
+ std::thread t0(predict, pred_hnds[0], image_data, nd_hnd, 0);
+ std::thread t1(predict, pred_hnds[1], image_data, nd_hnd, 1);
+ std::thread t2(predict, pred_hnds[2], image_data, nd_hnd, 2);
+ std::thread t3(predict, pred_hnds[3], image_data, nd_hnd, 3);
+ std::thread t4(predict, pred_hnds[4], image_data, nd_hnd, 4);
+ std::thread t5(predict, pred_hnds[5], image_data, nd_hnd, 5);
+ std::thread t6(predict, pred_hnds[6], image_data, nd_hnd, 6);
+ std::thread t7(predict, pred_hnds[7], image_data, nd_hnd, 7);
+ t0.join();
+ t1.join();
+ t2.join();
+ t3.join();
+ t4.join();
+ t5.join();
+ t6.join();
+ t7.join();
return EXIT_SUCCESS;
}
diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h
index cc1c2966bd7..a42ad70feed 100644
--- a/include/mxnet/c_predict_api.h
+++ b/include/mxnet/c_predict_api.h
@@ -119,6 +119,18 @@ MXNET_DLL int MXPredCreatePartialOut(const char* symbol_json_str,
mx_uint num_output_nodes,
const char** output_keys,
PredictorHandle* out);
+
+int MXPredCreateMultithread(const char* symbol_json_str,
+ const void* param_bytes,
+ int param_size,
+ int dev_type, int dev_id,
+ mx_uint num_input_nodes,
+ const char** input_keys,
+ const mx_uint* input_shape_indptr,
+ const mx_uint* input_shape_data,
+ int num_threads,
+ PredictorHandle* out);
+
/*!
* \brief Change the input shape of an existing predictor.
* \param num_input_nodes Number of input nodes to the net,
diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc
index d84a89ab213..a7753b1d617 100644
--- a/src/c_api/c_predict_api.cc
+++ b/src/c_api/c_predict_api.cc
@@ -67,34 +67,7 @@ struct MXAPINDList {
std::vector<mx_float> data;
};
-int MXPredCreate(const char* symbol_json_str,
- const void* param_bytes,
- int param_size,
- int dev_type, int dev_id,
- mx_uint num_input_nodes,
- const char** input_keys,
- const mx_uint* input_shape_indptr,
- const mx_uint* input_shape_data,
- PredictorHandle* out) {
- return MXPredCreatePartialOut(
- symbol_json_str,
- param_bytes,
- param_size,
- dev_type,
- dev_id,
- num_input_nodes,
- input_keys,
- input_shape_indptr,
- input_shape_data,
- 0,
- NULL,
- out);
-}
-namespace mxnet {
-
-} // namespace mxnet
-
-int MXPredCreatePartialOut(const char* symbol_json_str,
+int MXPredCreatePartialOut1(const char* symbol_json_str,
const void* param_bytes,
int param_size,
int dev_type, int dev_id,
@@ -104,10 +77,11 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
const mx_uint* input_shape_data,
mx_uint num_output_nodes,
const char** output_keys,
+ // This is used for paralle inference.
+ int num_threads,
PredictorHandle* out) {
using nnvm::Symbol;
- MXAPIPredictor* ret = new MXAPIPredictor();
API_BEGIN();
Symbol sym;
// make sure symbols are registered
@@ -140,7 +114,6 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
}
sym = nnvm::Symbol::CreateGroup(out_syms);
}
- ret->sym = sym;
// load the parameters
std::unordered_map<std::string, NDArray> arg_params, aux_params;
@@ -188,9 +161,10 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
std::vector<TShape> out_shapes(sym.ListOutputNames().size());
std::vector<TShape> aux_shapes(aux_names.size());
std::vector<TShape> arg_shapes;
+ std::unordered_map<std::string, size_t> key2arg;
for (size_t i = 0; i < arg_names.size(); ++i) {
std::string key = arg_names[i];
- ret->key2arg[key] = i;
+ key2arg[key] = i;
}
try {
@@ -215,7 +189,6 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
}
Context ctx = Context::Create(static_cast<Context::DeviceType>(dev_type), dev_id);
- ret->ctx = ctx;
std::vector<NDArray> arg_arrays, aux_arrays;
for (size_t i = 0; i < arg_shapes.size(); ++i) {
@@ -232,10 +205,15 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
}
aux_arrays.push_back(nd);
}
- ret->arg_arrays = arg_arrays;
- ret->aux_arrays = aux_arrays;
// bind
- {
+ for (int i = 0; i < num_threads; i++) {
+ std::unique_ptr<MXAPIPredictor> ret(new MXAPIPredictor());
+ ret->sym = sym;
+ ret->ctx = ctx;
+ ret->key2arg = key2arg;
+ ret->arg_arrays = arg_arrays;
+ ret->aux_arrays = aux_arrays;
+
std::map<std::string, Context> ctx_map;
std::vector<NDArray> grad_store(arg_arrays.size());
std::vector<OpReqType> grad_req(arg_arrays.size(), kNullOp);
@@ -247,9 +225,89 @@ int MXPredCreatePartialOut(const char* symbol_json_str,
aux_arrays));
ret->out_shapes = out_shapes;
ret->out_arrays = ret->exec->outputs();
+ out[i] = ret.release();
}
- *out = ret;
- API_END_HANDLE_ERROR(delete ret);
+ API_END_HANDLE_ERROR();
+}
+
+int MXPredCreatePartialOut(const char* symbol_json_str,
+ const void* param_bytes,
+ int param_size,
+ int dev_type, int dev_id,
+ mx_uint num_input_nodes,
+ const char** input_keys,
+ const mx_uint* input_shape_indptr,
+ const mx_uint* input_shape_data,
+ mx_uint num_output_nodes,
+ const char** output_keys,
+ // This is used for paralle inference.
+ int num_threads,
+ PredictorHandle* out) {
+ return MXPredCreatePartialOut1(
+ symbol_json_str,
+ param_bytes,
+ param_size,
+ dev_type, dev_id,
+ num_input_nodes,
+ input_keys,
+ input_shape_indptr,
+ input_shape_data,
+ num_output_nodes,
+ output_keys,
+ 1,
+ out);
+}
+
+int MXPredCreate(const char* symbol_json_str,
+ const void* param_bytes,
+ int param_size,
+ int dev_type, int dev_id,
+ mx_uint num_input_nodes,
+ const char** input_keys,
+ const mx_uint* input_shape_indptr,
+ const mx_uint* input_shape_data,
+ PredictorHandle* out) {
+ return MXPredCreatePartialOut1(
+ symbol_json_str,
+ param_bytes,
+ param_size,
+ dev_type,
+ dev_id,
+ num_input_nodes,
+ input_keys,
+ input_shape_indptr,
+ input_shape_data,
+ 0,
+ NULL,
+ 1,
+ out);
+}
+
+int MXPredCreateMultithread(const char* symbol_json_str,
+ const void* param_bytes,
+ int param_size,
+ int dev_type, int dev_id,
+ mx_uint num_input_nodes,
+ const char** input_keys,
+ const mx_uint* input_shape_indptr,
+ const mx_uint* input_shape_data,
+ // This is used for paralle inference.
+ int num_threads,
+ PredictorHandle* out) {
+ return MXPredCreatePartialOut1(
+ symbol_json_str,
+ param_bytes,
+ param_size,
+ dev_type,
+ dev_id,
+ num_input_nodes,
+ input_keys,
+ input_shape_indptr,
+ input_shape_data,
+ 0,
+ NULL,
+ num_threads,
+ out);
}
int MXPredReshape(mx_uint num_input_nodes,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services