You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2019/08/01 22:32:52 UTC
[incubator-mxnet] 02/02: import mshadow source tree
This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit 1434b98e26ace5300f17465fbb2942272a3dfd77
Author: Sheng Zha <zh...@amazon.com>
AuthorDate: Thu Jul 18 20:11:57 2019 -0700
import mshadow source tree
---
3rdparty/mshadow/.gitignore | 21 +
3rdparty/mshadow/.travis.yml | 43 +
3rdparty/mshadow/CHANGES.md | 12 +
3rdparty/mshadow/CMakeLists.txt | 6 +
3rdparty/mshadow/LICENSE | 13 +
3rdparty/mshadow/README.md | 37 +
3rdparty/mshadow/cmake/Cuda.cmake | 324 +++
3rdparty/mshadow/cmake/Utils.cmake | 398 ++++
3rdparty/mshadow/cmake/mshadow.cmake | 91 +
3rdparty/mshadow/cmake/mshadowUtils.cmake | 2 +
3rdparty/mshadow/doc/Doxyfile | 2358 ++++++++++++++++++++
3rdparty/mshadow/doc/README.md | 321 +++
3rdparty/mshadow/doc/mkdoc.sh | 4 +
3rdparty/mshadow/guide/.gitignore | 3 +
3rdparty/mshadow/guide/Makefile | 37 +
3rdparty/mshadow/guide/README.md | 226 ++
3rdparty/mshadow/guide/basic.cpp | 161 ++
3rdparty/mshadow/guide/basic_stream.cu | 35 +
3rdparty/mshadow/guide/config.mk | 39 +
3rdparty/mshadow/guide/defop.cpp | 49 +
3rdparty/mshadow/guide/exp-template/.gitignore | 1 +
3rdparty/mshadow/guide/exp-template/Makefile | 20 +
3rdparty/mshadow/guide/exp-template/README.md | 340 +++
3rdparty/mshadow/guide/exp-template/exp_lazy.cpp | 45 +
.../mshadow/guide/exp-template/exp_template.cpp | 72 +
.../mshadow/guide/exp-template/exp_template_op.cpp | 92 +
3rdparty/mshadow/guide/mshadow-ps/.gitignore | 4 +
3rdparty/mshadow/guide/mshadow-ps/2-levels.png | Bin 0 -> 59413 bytes
3rdparty/mshadow/guide/mshadow-ps/Makefile | 45 +
3rdparty/mshadow/guide/mshadow-ps/README.md | 227 ++
3rdparty/mshadow/guide/mshadow-ps/config.mk | 40 +
3rdparty/mshadow/guide/mshadow-ps/dbstr.h | 35 +
.../mshadow/guide/mshadow-ps/dist_async_sum-inl.h | 124 +
.../mshadow/guide/mshadow-ps/dist_async_sum.cpp | 11 +
3rdparty/mshadow/guide/mshadow-ps/local.sh | 39 +
3rdparty/mshadow/guide/mshadow-ps/local_sum-inl.h | 119 +
3rdparty/mshadow/guide/mshadow-ps/local_sum.cpp | 4 +
3rdparty/mshadow/guide/mshadow-ps/local_sum.cu | 4 +
3rdparty/mshadow/guide/neuralnet/Makefile | 38 +
3rdparty/mshadow/guide/neuralnet/README.md | 16 +
3rdparty/mshadow/guide/neuralnet/config.mk | 35 +
3rdparty/mshadow/guide/neuralnet/convnet.cu | 282 +++
3rdparty/mshadow/guide/neuralnet/nnet.cu | 202 ++
3rdparty/mshadow/guide/neuralnet/nnet_ps.cu | 312 +++
3rdparty/mshadow/guide/neuralnet/util.h | 86 +
3rdparty/mshadow/make/README.md | 18 +
3rdparty/mshadow/make/mshadow.mk | 166 ++
3rdparty/mshadow/mshadow-ps/.gitignore | 3 +
3rdparty/mshadow/mshadow-ps/README.md | 4 +
3rdparty/mshadow/mshadow-ps/mshadow_ps.h | 358 +++
3rdparty/mshadow/mshadow-ps/ps_dist-inl.h | 126 ++
3rdparty/mshadow/mshadow-ps/ps_local-inl.h | 814 +++++++
3rdparty/mshadow/mshadow-ps/ps_rabit-inl.h | 113 +
3rdparty/mshadow/mshadow-ps/thread.h | 261 +++
3rdparty/mshadow/mshadow-ps/thread_util.h | 169 ++
3rdparty/mshadow/mshadow/README.md | 8 +
3rdparty/mshadow/mshadow/base.h | 1110 +++++++++
3rdparty/mshadow/mshadow/cuda/reduce.cuh | 120 +
3rdparty/mshadow/mshadow/cuda/tensor_gpu-inl.cuh | 828 +++++++
3rdparty/mshadow/mshadow/dot_engine-inl.h | 936 ++++++++
3rdparty/mshadow/mshadow/expr_engine-inl.h | 482 ++++
3rdparty/mshadow/mshadow/expr_scalar-inl.h | 165 ++
3rdparty/mshadow/mshadow/expression.h | 416 ++++
3rdparty/mshadow/mshadow/extension.h | 41 +
3rdparty/mshadow/mshadow/extension/broadcast.h | 165 ++
.../mshadow/extension/broadcast_with_axis.h | 258 +++
3rdparty/mshadow/mshadow/extension/channel_pool.h | 108 +
.../mshadow/mshadow/extension/channel_unpool.h | 137 ++
3rdparty/mshadow/mshadow/extension/choose.h | 90 +
3rdparty/mshadow/mshadow/extension/complex.h | 525 +++++
3rdparty/mshadow/mshadow/extension/concat.h | 194 ++
3rdparty/mshadow/mshadow/extension/crop.h | 119 +
3rdparty/mshadow/mshadow/extension/fill.h | 103 +
3rdparty/mshadow/mshadow/extension/flip.h | 132 ++
3rdparty/mshadow/mshadow/extension/implicit_gemm.h | 128 ++
3rdparty/mshadow/mshadow/extension/mask.h | 97 +
3rdparty/mshadow/mshadow/extension/mirror.h | 62 +
3rdparty/mshadow/mshadow/extension/one_hot.h | 87 +
.../mshadow/mshadow/extension/pack_col2patch.h | 154 ++
3rdparty/mshadow/mshadow/extension/pad.h | 111 +
3rdparty/mshadow/mshadow/extension/range.h | 118 +
.../mshadow/mshadow/extension/reduce_with_axis.h | 136 ++
3rdparty/mshadow/mshadow/extension/reduceto1d.h | 104 +
3rdparty/mshadow/mshadow/extension/reshape.h | 87 +
3rdparty/mshadow/mshadow/extension/slice.h | 156 ++
3rdparty/mshadow/mshadow/extension/slice_ex.h | 135 ++
3rdparty/mshadow/mshadow/extension/spatial_pool.h | 152 ++
.../mshadow/mshadow/extension/spatial_unpool.h | 135 ++
.../mshadow/extension/spatial_upsampling_nearest.h | 71 +
3rdparty/mshadow/mshadow/extension/swapaxis.h | 110 +
3rdparty/mshadow/mshadow/extension/take.h | 99 +
3rdparty/mshadow/mshadow/extension/take_grad.h | 111 +
3rdparty/mshadow/mshadow/extension/transpose.h | 200 ++
.../mshadow/mshadow/extension/unpack_patch2col.h | 151 ++
3rdparty/mshadow/mshadow/half.h | 354 +++
3rdparty/mshadow/mshadow/half2.h | 143 ++
3rdparty/mshadow/mshadow/io.h | 137 ++
3rdparty/mshadow/mshadow/logging.h | 234 ++
3rdparty/mshadow/mshadow/packet-inl.h | 413 ++++
3rdparty/mshadow/mshadow/packet/plain-inl.h | 76 +
3rdparty/mshadow/mshadow/packet/sse-inl.h | 147 ++
3rdparty/mshadow/mshadow/random.h | 570 +++++
3rdparty/mshadow/mshadow/stream_gpu-inl.h | 214 ++
3rdparty/mshadow/mshadow/tensor.h | 1081 +++++++++
3rdparty/mshadow/mshadow/tensor_container.h | 208 ++
3rdparty/mshadow/mshadow/tensor_cpu-inl.h | 627 ++++++
3rdparty/mshadow/mshadow/tensor_gpu-inl.h | 245 ++
3rdparty/mshadow/scripts/travis_script.sh | 19 +
3rdparty/mshadow/test/Makefile | 35 +
3rdparty/mshadow/test/pairtest.cu | 105 +
3rdparty/mshadow/test/pool.cu | 69 +
3rdparty/mshadow/test/reshape.cu | 74 +
3rdparty/mshadow/test/test.cu | 79 +
3rdparty/mshadow/test/test.h | 67 +
3rdparty/mshadow/test/unpack.cu | 85 +
115 files changed, 21728 insertions(+)
diff --git a/3rdparty/mshadow/.gitignore b/3rdparty/mshadow/.gitignore
new file mode 100644
index 0000000..3da5172
--- /dev/null
+++ b/3rdparty/mshadow/.gitignore
@@ -0,0 +1,21 @@
+# Compiled Object files
+*.slo
+*.lo
+*.o
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*~
+doc/html
+doc/latex
+rabit
+dmlc-core
+*.db
+*.bak
+build
diff --git a/3rdparty/mshadow/.travis.yml b/3rdparty/mshadow/.travis.yml
new file mode 100644
index 0000000..a4d6223
--- /dev/null
+++ b/3rdparty/mshadow/.travis.yml
@@ -0,0 +1,43 @@
+# disable sudo to use container based build
+sudo: false
+
+# Use Build Matrix to do lint and build seperately
+env:
+ matrix:
+ - TASK=lint LINT_LANG=cpp
+ - TASK=doc
+ - TASK=build CXX=g++
+
+# dependent apt packages
+addons:
+ apt:
+ packages:
+ - doxygen
+ - wget
+ - unzip
+ - libblas-dev
+ - python3-pip
+
+before_install:
+ - git clone https://github.com/dmlc/dmlc-core
+ - export TRAVIS=dmlc-core/scripts/travis
+ - source ${TRAVIS}/travis_setup_env.sh
+
+install:
+ - pip3 install --upgrade pip --user
+ - pip3 install --user cpplint pylint
+
+script: scripts/travis_script.sh
+
+before_cache:
+ - ${TRAVIS}/travis_before_cache.sh
+
+cache:
+ directories:
+ - ${HOME}/.cache/usr
+
+notifications:
+ email:
+ on_success: change
+ on_failure: always
+
diff --git a/3rdparty/mshadow/CHANGES.md b/3rdparty/mshadow/CHANGES.md
new file mode 100644
index 0000000..03bb169
--- /dev/null
+++ b/3rdparty/mshadow/CHANGES.md
@@ -0,0 +1,12 @@
+Change Log
+=====
+
+mshadow-1.0
+=====
+* Initial release
+
+mshadow-2.0: in progress
+=====
+* Support multiple data type
+* Great refactoring of code
+* Parameter server interface for MultiGPU and distributed learning
diff --git a/3rdparty/mshadow/CMakeLists.txt b/3rdparty/mshadow/CMakeLists.txt
new file mode 100644
index 0000000..b89e902
--- /dev/null
+++ b/3rdparty/mshadow/CMakeLists.txt
@@ -0,0 +1,6 @@
+cmake_minimum_required(VERSION 2.8.7)
+
+project(mshadow C CXX)
+
+set(mshadow_LINT_DIRS mshadow mshadow-ps)
+add_custom_target(mshadow_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${mshadow_LINT_DIRS} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -DPROJECT_NAME=mshadow -P ${PROJECT_SOURCE_DIR}/../dmlc-core/cmake/lint.cmake)
diff --git a/3rdparty/mshadow/LICENSE b/3rdparty/mshadow/LICENSE
new file mode 100644
index 0000000..ebf9611
--- /dev/null
+++ b/3rdparty/mshadow/LICENSE
@@ -0,0 +1,13 @@
+Copyright (c) 2014 by Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/3rdparty/mshadow/README.md b/3rdparty/mshadow/README.md
new file mode 100644
index 0000000..cc18964
--- /dev/null
+++ b/3rdparty/mshadow/README.md
@@ -0,0 +1,37 @@
+mshadow: Matrix Shadow
+======
+[](https://travis-ci.org/dmlc/mshadow)
+
+MShadow is a lightweight CPU/GPU Matrix/Tensor Template Library in C++/CUDA. The goal of mshadow is to support ***efficient***,
+***device invariant*** and ***simple*** tensor library for machine learning project that aims for maximum performance and control, while also emphasize simplicity.
+
+MShadow also provides interface that allows writing Multi-GPU and distributed deep learning programs in an easy and unified way.
+
+* [Contributors](https://github.com/tqchen/mshadow/graphs/contributors)
+* [Tutorial](guide)
+* [Documentation](doc)
+* [Parameter Server Interface for GPU Tensor](guide/mshadow-ps)
+
+Features
+--------
+* Efficient: all the expression you write will be lazily evaluated and compiled into optimized code
+ - No temporal memory allocation will happen for expression you write
+ - mshadow will generate specific kernel for every expression you write in compile time.
+* Device invariant: you can write one code and it will run on both CPU and GPU
+* Simple: mshadow allows you to write machine learning code using expressions.
+* Whitebox: put a float* into the Tensor struct and take the benefit of the package, no memory allocation is happened unless explicitly called
+* Lightweight library: light amount of code to support frequently used functions in machine learning
+* Extendable: user can write simple functions that plugs into mshadow and run on GPU/CPU, no experience in CUDA is required.
+* MultiGPU and Distributed ML: mshadow-ps interface allows user to write efficient MultiGPU and distributed programs in an unified way.
+
+Version
+-------
+* This version mshadow-2.x, there are a lot of changes in the interface and it is not backward compatible with mshadow-1.0
+ - If you use older version of cxxnet, you will need to use the legacy mshadow code
+* For legacy code, refer to [Here](https://github.com/tqchen/mshadow/releases/tag/v1.1)
+* Change log in [CHANGES.md](CHANGES.md)
+
+Projects Using MShadow
+----------------------
+* [MXNet: Efficient and Flexible Distributed Deep Learning Framework](https://github.com/dmlc/mxnet)
+* [CXXNet: A lightweight C++ based deep learnig framework](https://github.com/dmlc/cxxnet)
diff --git a/3rdparty/mshadow/cmake/Cuda.cmake b/3rdparty/mshadow/cmake/Cuda.cmake
new file mode 100644
index 0000000..bc09a39
--- /dev/null
+++ b/3rdparty/mshadow/cmake/Cuda.cmake
@@ -0,0 +1,324 @@
+if(NOT USE_CUDA)
+ return()
+endif()
+
+include(CheckCXXCompilerFlag)
+check_cxx_compiler_flag("-std=c++11" SUPPORT_CXX11)
+
+################################################################################################
+# A function for automatic detection of GPUs installed (if autodetection is enabled)
+# Usage:
+# mshadow_detect_installed_gpus(out_variable)
+function(mshadow_detect_installed_gpus out_variable)
+set(CUDA_gpu_detect_output "")
+ if(NOT CUDA_gpu_detect_output)
+ message(STATUS "Running GPU architecture autodetection")
+ set(__cufile ${PROJECT_BINARY_DIR}/detect_cuda_archs.cu)
+
+ file(WRITE ${__cufile} ""
+ "#include <cstdio>\n"
+ "#include <iostream>\n"
+ "using namespace std;\n"
+ "int main()\n"
+ "{\n"
+ " int count = 0;\n"
+ " if (cudaSuccess != cudaGetDeviceCount(&count)) { return -1; }\n"
+ " if (count == 0) { cerr << \"No cuda devices detected\" << endl; return -1; }\n"
+ " for (int device = 0; device < count; ++device)\n"
+ " {\n"
+ " cudaDeviceProp prop;\n"
+ " if (cudaSuccess == cudaGetDeviceProperties(&prop, device))\n"
+ " std::printf(\"%d.%d \", prop.major, prop.minor);\n"
+ " }\n"
+ " return 0;\n"
+ "}\n")
+ if(MSVC)
+ #find vcvarsall.bat and run it building msvc environment
+ get_filename_component(MY_COMPILER_DIR ${CMAKE_CXX_COMPILER} DIRECTORY)
+ find_file(MY_VCVARSALL_BAT vcvarsall.bat "${MY_COMPILER_DIR}/.." "${MY_COMPILER_DIR}/../..")
+ execute_process(COMMAND ${MY_VCVARSALL_BAT} && ${CUDA_NVCC_EXECUTABLE} -arch sm_30 --run ${__cufile}
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/"
+ RESULT_VARIABLE __nvcc_res OUTPUT_VARIABLE __nvcc_out
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+ else()
+ if(CUDA_LIBRARY_PATH)
+ set(CUDA_LINK_LIBRARY_PATH "-L${CUDA_LIBRARY_PATH}")
+ endif()
+ execute_process(COMMAND ${CUDA_NVCC_EXECUTABLE} -arch sm_30 --run ${__cufile} ${CUDA_LINK_LIBRARY_PATH}
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/"
+ RESULT_VARIABLE __nvcc_res OUTPUT_VARIABLE __nvcc_out
+ OUTPUT_STRIP_TRAILING_WHITESPACE)
+ endif()
+ if(__nvcc_res EQUAL 0)
+ # nvcc outputs text containing line breaks when building with MSVC.
+ # The line below prevents CMake from inserting a variable with line
+ # breaks in the cache
+ message(STATUS "Found CUDA arch ${__nvcc_out}")
+ string(REGEX MATCH "([1-9].[0-9])" __nvcc_out "${__nvcc_out}")
+ string(REPLACE "2.1" "2.1(2.0)" __nvcc_out "${__nvcc_out}")
+ set(CUDA_gpu_detect_output ${__nvcc_out} CACHE INTERNAL "Returned GPU architetures from mshadow_detect_gpus tool" FORCE)
+ else()
+ message(WARNING "Running GPU detection script with nvcc failed: ${__nvcc_out}")
+ endif()
+ endif()
+
+ if(NOT CUDA_gpu_detect_output)
+ message(WARNING "Automatic GPU detection failed. Building for all known architectures (${mshadow_known_gpu_archs}).")
+ set(${out_variable} ${mshadow_known_gpu_archs} PARENT_SCOPE)
+ else()
+ set(${out_variable} ${CUDA_gpu_detect_output} PARENT_SCOPE)
+ endif()
+endfunction()
+
+
+################################################################################################
+# Function for selecting GPU arch flags for nvcc based on CUDA_ARCH_NAME
+# Usage:
+# mshadow_select_nvcc_arch_flags(out_variable)
+function(mshadow_select_nvcc_arch_flags out_variable)
+ # List of arch names
+ set(__archs_names "Fermi" "Kepler" "Maxwell" "Pascal" "Volta" "All" "Manual")
+ set(__archs_name_default "All")
+ if(NOT CMAKE_CROSSCOMPILING)
+ list(APPEND __archs_names "Auto")
+ set(__archs_name_default "Auto")
+ endif()
+
+ # set CUDA_ARCH_NAME strings (so it will be seen as dropbox in CMake-Gui)
+ set(CUDA_ARCH_NAME ${__archs_name_default} CACHE STRING "Select target NVIDIA GPU achitecture.")
+ set_property( CACHE CUDA_ARCH_NAME PROPERTY STRINGS "" ${__archs_names} )
+ mark_as_advanced(CUDA_ARCH_NAME)
+
+ # verify CUDA_ARCH_NAME value
+ if(NOT ";${__archs_names};" MATCHES ";${CUDA_ARCH_NAME};")
+ string(REPLACE ";" ", " __archs_names "${__archs_names}")
+ message(FATAL_ERROR "Only ${__archs_names} architeture names are supported.")
+ endif()
+
+ if(${CUDA_ARCH_NAME} STREQUAL "Manual")
+ set(CUDA_ARCH_BIN ${mshadow_known_gpu_archs} CACHE STRING "Specify 'real' GPU architectures to build binaries for, BIN(PTX) format is supported")
+ set(CUDA_ARCH_PTX "50" CACHE STRING "Specify 'virtual' PTX architectures to build PTX intermediate code for")
+ mark_as_advanced(CUDA_ARCH_BIN CUDA_ARCH_PTX)
+ else()
+ unset(CUDA_ARCH_BIN CACHE)
+ unset(CUDA_ARCH_PTX CACHE)
+ endif()
+
+ if(${CUDA_ARCH_NAME} STREQUAL "Fermi")
+ set(__cuda_arch_bin "20 21(20)")
+ elseif(${CUDA_ARCH_NAME} STREQUAL "Kepler")
+ set(__cuda_arch_bin "30 35")
+ elseif(${CUDA_ARCH_NAME} STREQUAL "Maxwell")
+ set(__cuda_arch_bin "50")
+ elseif(${CUDA_ARCH_NAME} STREQUAL "Pascal")
+ set(__cuda_arch_bin "60 61")
+ elseif(${CUDA_ARCH_NAME} STREQUAL "Volta")
+ set(__cuda_arch_bin "70")
+ elseif(${CUDA_ARCH_NAME} STREQUAL "All")
+ set(__cuda_arch_bin ${mshadow_known_gpu_archs})
+ elseif(${CUDA_ARCH_NAME} STREQUAL "Auto")
+ mshadow_detect_installed_gpus(__cuda_arch_bin)
+ else() # (${CUDA_ARCH_NAME} STREQUAL "Manual")
+ set(__cuda_arch_bin ${CUDA_ARCH_BIN})
+ endif()
+
+ # remove dots and convert to lists
+ string(REGEX REPLACE "\\." "" __cuda_arch_bin "${__cuda_arch_bin}")
+ string(REGEX REPLACE "\\." "" __cuda_arch_ptx "${CUDA_ARCH_PTX}")
+ string(REGEX MATCHALL "[0-9()]+" __cuda_arch_bin "${__cuda_arch_bin}")
+ string(REGEX MATCHALL "[0-9]+" __cuda_arch_ptx "${__cuda_arch_ptx}")
+ mshadow_list_unique(__cuda_arch_bin __cuda_arch_ptx)
+
+ set(__nvcc_flags "")
+ set(__nvcc_archs_readable "")
+
+ # Tell NVCC to add binaries for the specified GPUs
+ foreach(__arch ${__cuda_arch_bin})
+ if(__arch MATCHES "([0-9]+)\\(([0-9]+)\\)")
+ # User explicitly specified PTX for the concrete BIN
+ list(APPEND __nvcc_flags -gencode arch=compute_${CMAKE_MATCH_2},code=sm_${CMAKE_MATCH_1})
+ list(APPEND __nvcc_archs_readable sm_${CMAKE_MATCH_1})
+ else()
+ # User didn't explicitly specify PTX for the concrete BIN, we assume PTX=BIN
+ list(APPEND __nvcc_flags -gencode arch=compute_${__arch},code=sm_${__arch})
+ list(APPEND __nvcc_archs_readable sm_${__arch})
+ endif()
+ endforeach()
+
+ # Tell NVCC to add PTX intermediate code for the specified architectures
+ foreach(__arch ${__cuda_arch_ptx})
+ list(APPEND __nvcc_flags -gencode arch=compute_${__arch},code=compute_${__arch})
+ list(APPEND __nvcc_archs_readable compute_${__arch})
+ endforeach()
+
+ string(REPLACE ";" " " __nvcc_archs_readable "${__nvcc_archs_readable}")
+ set(${out_variable} ${__nvcc_flags} PARENT_SCOPE)
+ set(${out_variable}_readable ${__nvcc_archs_readable} PARENT_SCOPE)
+endfunction()
+
+################################################################################################
+# Short command for cuda comnpilation
+# Usage:
+# mshadow_cuda_compile(<objlist_variable> <cuda_files>)
+macro(mshadow_cuda_compile objlist_variable)
+ foreach(var CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG)
+ set(${var}_backup_in_cuda_compile_ "${${var}}")
+
+ # we remove /EHa as it generates warnings under windows
+ string(REPLACE "/EHa" "" ${var} "${${var}}")
+
+ endforeach()
+ if(UNIX OR APPLE)
+ list(APPEND CUDA_NVCC_FLAGS -Xcompiler -fPIC)
+ endif()
+
+ if(APPLE)
+ list(APPEND CUDA_NVCC_FLAGS -Xcompiler -Wno-unused-function)
+ endif()
+
+ set(CUDA_NVCC_FLAGS_DEBUG "${CUDA_NVCC_FLAGS_DEBUG} -G")
+
+ if(MSVC)
+ # disable noisy warnings:
+ # 4819: The file contains a character that cannot be represented in the current code page (number).
+ list(APPEND CUDA_NVCC_FLAGS -Xcompiler "/wd4819")
+ foreach(flag_var
+ CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
+ CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+ if(${flag_var} MATCHES "/MD")
+ string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
+ endif(${flag_var} MATCHES "/MD")
+ endforeach(flag_var)
+ endif()
+
+ # If the build system is a container, make sure the nvcc intermediate files
+ # go into the build output area rather than in /tmp, which may run out of space
+ if(IS_CONTAINER_BUILD)
+ set(CUDA_NVCC_INTERMEDIATE_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+ message(STATUS "Container build enabled, so nvcc intermediate files in: ${CUDA_NVCC_INTERMEDIATE_DIR}")
+ list(APPEND CUDA_NVCC_FLAGS "--keep --keep-dir ${CUDA_NVCC_INTERMEDIATE_DIR}")
+ endif()
+
+ cuda_compile(cuda_objcs ${ARGN})
+
+ foreach(var CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_DEBUG)
+ set(${var} "${${var}_backup_in_cuda_compile_}")
+ unset(${var}_backup_in_cuda_compile_)
+ endforeach()
+
+ set(${objlist_variable} ${cuda_objcs})
+endmacro()
+
+################################################################################################
+# Short command for cuDNN detection. Believe it soon will be a part of CUDA toolkit distribution.
+# That's why not FindcuDNN.cmake file, but just the macro
+# Usage:
+# detect_cuDNN()
+function(detect_cuDNN)
+ set(CUDNN_ROOT "" CACHE PATH "CUDNN root folder")
+
+ find_path(CUDNN_INCLUDE cudnn.h
+ PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDA_TOOLKIT_INCLUDE}
+ DOC "Path to cuDNN include directory." )
+
+ get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH)
+ find_library(CUDNN_LIBRARY NAMES libcudnn.so cudnn.lib # libcudnn_static.a
+ PATHS ${CUDNN_ROOT} $ENV{CUDNN_ROOT} ${CUDNN_INCLUDE} ${__libpath_hist}
+ DOC "Path to cuDNN library.")
+
+ if(CUDNN_INCLUDE AND CUDNN_LIBRARY)
+ set(HAVE_CUDNN TRUE PARENT_SCOPE)
+ set(CUDNN_FOUND TRUE PARENT_SCOPE)
+
+ mark_as_advanced(CUDNN_INCLUDE CUDNN_LIBRARY CUDNN_ROOT)
+ message(STATUS "Found cuDNN (include: ${CUDNN_INCLUDE}, library: ${CUDNN_LIBRARY})")
+ endif()
+endfunction()
+
+
+################################################################################################
+### Non macro section
+################################################################################################
+
+# Try to prime CUDA_TOOLKIT_ROOT_DIR by looking for libcudart.so
+if(NOT CUDA_TOOLKIT_ROOT_DIR)
+ find_library(CUDA_LIBRARY_PATH libcudart.so PATHS ENV LD_LIBRARY_PATH PATH_SUFFIXES lib lib64)
+ if(CUDA_LIBRARY_PATH)
+ get_filename_component(CUDA_LIBRARY_PATH ${CUDA_LIBRARY_PATH} DIRECTORY)
+ set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_LIBRARY_PATH}/..")
+ endif()
+endif()
+
+find_package(CUDA 5.5 QUIET REQUIRED)
+find_cuda_helper_libs(curand) # cmake 2.8.7 compartibility which doesn't search for curand
+
+if(NOT CUDA_FOUND)
+ return()
+endif()
+
+set(HAVE_CUDA TRUE)
+message(STATUS "CUDA detected: " ${CUDA_VERSION})
+include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
+list(APPEND mshadow_LINKER_LIBS ${CUDA_CUDART_LIBRARY}
+ ${CUDA_curand_LIBRARY} ${CUDA_CUBLAS_LIBRARIES})
+
+# Known NVIDIA GPU achitectures mshadow can be compiled for.
+# This list will be used for CUDA_ARCH_NAME = All option
+if(CUDA_ARCH_ALL)
+ set(mshadow_known_gpu_archs "${CUDA_ARCH_ALL}")
+else()
+ if(${CUDA_VERSION} EQUAL 9.0 OR ${CUDA_VERSION} GREATER 9.0)
+ set(mshadow_known_gpu_archs "30 35 50 52 60 61 70")
+ elseif(${CUDA_VERSION} EQUAL 8.0 OR ${CUDA_VERSION} GREATER 8.0)
+ set(mshadow_known_gpu_archs "30 35 50 52 60 61")
+ else()
+ set(mshadow_known_gpu_archs "30 35 50 52")
+ endif()
+endif()
+
+# cudnn detection
+if(USE_CUDNN)
+ detect_cuDNN()
+ if(HAVE_CUDNN)
+ add_definitions(-DUSE_CUDNN)
+ include_directories(SYSTEM ${CUDNN_INCLUDE})
+ list(APPEND mshadow_LINKER_LIBS ${CUDNN_LIBRARY})
+ endif()
+endif()
+
+# setting nvcc arch flags
+mshadow_select_nvcc_arch_flags(NVCC_FLAGS_EXTRA)
+list(APPEND CUDA_NVCC_FLAGS ${NVCC_FLAGS_EXTRA})
+message(STATUS "Added CUDA NVCC flags for: ${NVCC_FLAGS_EXTRA_readable}")
+
+# Boost 1.55 workaround, see https://svn.boost.org/trac/boost/ticket/9392 or
+# https://github.com/ComputationalRadiationPhysics/picongpu/blob/master/src/picongpu/CMakeLists.txt
+if(Boost_VERSION EQUAL 105500)
+ message(STATUS "Cuda + Boost 1.55: Applying noinline work around")
+ # avoid warning for CMake >= 2.8.12
+ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} \"-DBOOST_NOINLINE=__attribute__((noinline))\" ")
+endif()
+
+# disable some nvcc diagnostic that apears in boost, glog, glags, opencv, etc.
+foreach(diag cc_clobber_ignored integer_sign_change useless_using_declaration set_but_not_used)
+ list(APPEND CUDA_NVCC_FLAGS -Xcudafe --diag_suppress=${diag})
+endforeach()
+
+# setting default testing device
+if(NOT CUDA_TEST_DEVICE)
+ set(CUDA_TEST_DEVICE -1)
+endif()
+
+mark_as_advanced(CUDA_BUILD_CUBIN CUDA_BUILD_EMULATION CUDA_VERBOSE_BUILD)
+mark_as_advanced(CUDA_SDK_ROOT_DIR CUDA_SEPARABLE_COMPILATION)
+
+# Handle clang/libc++ issue
+if(APPLE)
+ mshadow_detect_darwin_version(OSX_VERSION)
+
+ # OSX 10.9 and higher uses clang/libc++ by default which is incompartible with old CUDA toolkits
+ if(OSX_VERSION VERSION_GREATER 10.8)
+ # enabled by default if and only if CUDA version is less than 7.0
+ mshadow_option(USE_libstdcpp "Use libstdc++ instead of libc++" (CUDA_VERSION VERSION_LESS 7.0))
+ endif()
+endif()
diff --git a/3rdparty/mshadow/cmake/Utils.cmake b/3rdparty/mshadow/cmake/Utils.cmake
new file mode 100644
index 0000000..dc464f0
--- /dev/null
+++ b/3rdparty/mshadow/cmake/Utils.cmake
@@ -0,0 +1,398 @@
+################################################################################################
+# Command alias for debugging messages
+# Usage:
+# dmsg(<message>)
+function(dmsg)
+ message(STATUS ${ARGN})
+endfunction()
+
+################################################################################################
+# Removes duplicates from list(s)
+# Usage:
+# mshadow_list_unique(<list_variable> [<list_variable>] [...])
+macro(mshadow_list_unique)
+ foreach(__lst ${ARGN})
+ if(${__lst})
+ list(REMOVE_DUPLICATES ${__lst})
+ endif()
+ endforeach()
+endmacro()
+
+################################################################################################
+# Clears variables from list
+# Usage:
+# mshadow_clear_vars(<variables_list>)
+macro(mshadow_clear_vars)
+ foreach(_var ${ARGN})
+ unset(${_var})
+ endforeach()
+endmacro()
+
+################################################################################################
+# Removes duplicates from string
+# Usage:
+# mshadow_string_unique(<string_variable>)
+function(mshadow_string_unique __string)
+ if(${__string})
+ set(__list ${${__string}})
+ separate_arguments(__list)
+ list(REMOVE_DUPLICATES __list)
+ foreach(__e ${__list})
+ set(__str "${__str} ${__e}")
+ endforeach()
+ set(${__string} ${__str} PARENT_SCOPE)
+ endif()
+endfunction()
+
+################################################################################################
+# Prints list element per line
+# Usage:
+# mshadow_print_list(<list>)
+function(mshadow_print_list)
+ foreach(e ${ARGN})
+ message(STATUS ${e})
+ endforeach()
+endfunction()
+
+################################################################################################
+# Function merging lists of compiler flags to single string.
+# Usage:
+# mshadow_merge_flag_lists(out_variable <list1> [<list2>] [<list3>] ...)
+function(mshadow_merge_flag_lists out_var)
+ set(__result "")
+ foreach(__list ${ARGN})
+ foreach(__flag ${${__list}})
+ string(STRIP ${__flag} __flag)
+ set(__result "${__result} ${__flag}")
+ endforeach()
+ endforeach()
+ string(STRIP ${__result} __result)
+ set(${out_var} ${__result} PARENT_SCOPE)
+endfunction()
+
+################################################################################################
+# Converts all paths in list to absolute
+# Usage:
+# mshadow_convert_absolute_paths(<list_variable>)
+function(mshadow_convert_absolute_paths variable)
+ set(__dlist "")
+ foreach(__s ${${variable}})
+ get_filename_component(__abspath ${__s} ABSOLUTE)
+ list(APPEND __list ${__abspath})
+ endforeach()
+ set(${variable} ${__list} PARENT_SCOPE)
+endfunction()
+
+################################################################################################
+# Reads set of version defines from the header file
+# Usage:
+# mshadow_parse_header(<file> <define1> <define2> <define3> ..)
+macro(mshadow_parse_header FILENAME FILE_VAR)
+ set(vars_regex "")
+ set(__parnet_scope OFF)
+ set(__add_cache OFF)
+ foreach(name ${ARGN})
+ if("${name}" STREQUAL "PARENT_SCOPE")
+ set(__parnet_scope ON)
+ elseif("${name}" STREQUAL "CACHE")
+ set(__add_cache ON)
+ elseif(vars_regex)
+ set(vars_regex "${vars_regex}|${name}")
+ else()
+ set(vars_regex "${name}")
+ endif()
+ endforeach()
+ if(EXISTS "${FILENAME}")
+ file(STRINGS "${FILENAME}" ${FILE_VAR} REGEX "#define[ \t]+(${vars_regex})[ \t]+[0-9]+" )
+ else()
+ unset(${FILE_VAR})
+ endif()
+ foreach(name ${ARGN})
+ if(NOT "${name}" STREQUAL "PARENT_SCOPE" AND NOT "${name}" STREQUAL "CACHE")
+ if(${FILE_VAR})
+ if(${FILE_VAR} MATCHES ".+[ \t]${name}[ \t]+([0-9]+).*")
+ string(REGEX REPLACE ".+[ \t]${name}[ \t]+([0-9]+).*" "\\1" ${name} "${${FILE_VAR}}")
+ else()
+ set(${name} "")
+ endif()
+ if(__add_cache)
+ set(${name} ${${name}} CACHE INTERNAL "${name} parsed from ${FILENAME}" FORCE)
+ elseif(__parnet_scope)
+ set(${name} "${${name}}" PARENT_SCOPE)
+ endif()
+ else()
+ unset(${name} CACHE)
+ endif()
+ endif()
+ endforeach()
+endmacro()
+
+################################################################################################
+# Reads single version define from the header file and parses it
+# Usage:
+# mshadow_parse_header_single_define(<library_name> <file> <define_name>)
+function(mshadow_parse_header_single_define LIBNAME HDR_PATH VARNAME)
+ set(${LIBNAME}_H "")
+ if(EXISTS "${HDR_PATH}")
+ file(STRINGS "${HDR_PATH}" ${LIBNAME}_H REGEX "^#define[ \t]+${VARNAME}[ \t]+\"[^\"]*\".*$" LIMIT_COUNT 1)
+ endif()
+
+ if(${LIBNAME}_H)
+ string(REGEX REPLACE "^.*[ \t]${VARNAME}[ \t]+\"([0-9]+).*$" "\\1" ${LIBNAME}_VERSION_MAJOR "${${LIBNAME}_H}")
+ string(REGEX REPLACE "^.*[ \t]${VARNAME}[ \t]+\"[0-9]+\\.([0-9]+).*$" "\\1" ${LIBNAME}_VERSION_MINOR "${${LIBNAME}_H}")
+ string(REGEX REPLACE "^.*[ \t]${VARNAME}[ \t]+\"[0-9]+\\.[0-9]+\\.([0-9]+).*$" "\\1" ${LIBNAME}_VERSION_PATCH "${${LIBNAME}_H}")
+ set(${LIBNAME}_VERSION_MAJOR ${${LIBNAME}_VERSION_MAJOR} ${ARGN} PARENT_SCOPE)
+ set(${LIBNAME}_VERSION_MINOR ${${LIBNAME}_VERSION_MINOR} ${ARGN} PARENT_SCOPE)
+ set(${LIBNAME}_VERSION_PATCH ${${LIBNAME}_VERSION_PATCH} ${ARGN} PARENT_SCOPE)
+ set(${LIBNAME}_VERSION_STRING "${${LIBNAME}_VERSION_MAJOR}.${${LIBNAME}_VERSION_MINOR}.${${LIBNAME}_VERSION_PATCH}" PARENT_SCOPE)
+
+ # append a TWEAK version if it exists:
+ set(${LIBNAME}_VERSION_TWEAK "")
+ if("${${LIBNAME}_H}" MATCHES "^.*[ \t]${VARNAME}[ \t]+\"[0-9]+\\.[0-9]+\\.[0-9]+\\.([0-9]+).*$")
+ set(${LIBNAME}_VERSION_TWEAK "${CMAKE_MATCH_1}" ${ARGN} PARENT_SCOPE)
+ endif()
+ if(${LIBNAME}_VERSION_TWEAK)
+ set(${LIBNAME}_VERSION_STRING "${${LIBNAME}_VERSION_STRING}.${${LIBNAME}_VERSION_TWEAK}" ${ARGN} PARENT_SCOPE)
+ else()
+ set(${LIBNAME}_VERSION_STRING "${${LIBNAME}_VERSION_STRING}" ${ARGN} PARENT_SCOPE)
+ endif()
+ endif()
+endfunction()
+
+########################################################################################################
+# An option that the user can select. Can accept condition to control when option is available for user.
+# Usage:
+# mshadow_option(<option_variable> "doc string" <initial value or boolean expression> [IF <condition>])
+function(mshadow_option variable description value)
+ set(__value ${value})
+ set(__condition "")
+ set(__varname "__value")
+ foreach(arg ${ARGN})
+ if(arg STREQUAL "IF" OR arg STREQUAL "if")
+ set(__varname "__condition")
+ else()
+ list(APPEND ${__varname} ${arg})
+ endif()
+ endforeach()
+ unset(__varname)
+ if("${__condition}" STREQUAL "")
+ set(__condition 2 GREATER 1)
+ endif()
+
+ if(${__condition})
+ if("${__value}" MATCHES ";")
+ if(${__value})
+ option(${variable} "${description}" ON)
+ else()
+ option(${variable} "${description}" OFF)
+ endif()
+ elseif(DEFINED ${__value})
+ if(${__value})
+ option(${variable} "${description}" ON)
+ else()
+ option(${variable} "${description}" OFF)
+ endif()
+ else()
+ option(${variable} "${description}" ${__value})
+ endif()
+ else()
+ unset(${variable} CACHE)
+ endif()
+endfunction()
+
+################################################################################################
+# Utility macro for comparing two lists. Used for CMake debugging purposes
+# Usage:
+# mshadow_compare_lists(<list_variable> <list2_variable> [description])
+function(mshadow_compare_lists list1 list2 desc)
+ set(__list1 ${${list1}})
+ set(__list2 ${${list2}})
+ list(SORT __list1)
+ list(SORT __list2)
+ list(LENGTH __list1 __len1)
+ list(LENGTH __list2 __len2)
+
+ if(NOT ${__len1} EQUAL ${__len2})
+ message(FATAL_ERROR "Lists are not equal. ${__len1} != ${__len2}. ${desc}")
+ endif()
+
+ foreach(__i RANGE 1 ${__len1})
+ math(EXPR __index "${__i}- 1")
+ list(GET __list1 ${__index} __item1)
+ list(GET __list2 ${__index} __item2)
+ if(NOT ${__item1} STREQUAL ${__item2})
+ message(FATAL_ERROR "Lists are not equal. Differ at element ${__index}. ${desc}")
+ endif()
+ endforeach()
+endfunction()
+
+################################################################################################
+# Command for disabling warnings for different platforms (see below for gcc and VisualStudio)
+# Usage:
+# mshadow_warnings_disable(<CMAKE_[C|CXX]_FLAGS[_CONFIGURATION]> -Wshadow /wd4996 ..,)
+macro(mshadow_warnings_disable)
+ set(_flag_vars "")
+ set(_msvc_warnings "")
+ set(_gxx_warnings "")
+
+ foreach(arg ${ARGN})
+ if(arg MATCHES "^CMAKE_")
+ list(APPEND _flag_vars ${arg})
+ elseif(arg MATCHES "^/wd")
+ list(APPEND _msvc_warnings ${arg})
+ elseif(arg MATCHES "^-W")
+ list(APPEND _gxx_warnings ${arg})
+ endif()
+ endforeach()
+
+ if(NOT _flag_vars)
+ set(_flag_vars CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
+ endif()
+
+ if(MSVC AND _msvc_warnings)
+ foreach(var ${_flag_vars})
+ foreach(warning ${_msvc_warnings})
+ set(${var} "${${var}} ${warning}")
+ endforeach()
+ endforeach()
+ elseif((CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX) AND _gxx_warnings)
+ foreach(var ${_flag_vars})
+ foreach(warning ${_gxx_warnings})
+ if(NOT warning MATCHES "^-Wno-")
+ string(REPLACE "${warning}" "" ${var} "${${var}}")
+ string(REPLACE "-W" "-Wno-" warning "${warning}")
+ endif()
+ set(${var} "${${var}} ${warning}")
+ endforeach()
+ endforeach()
+ endif()
+ mshadow_clear_vars(_flag_vars _msvc_warnings _gxx_warnings)
+endmacro()
+
+################################################################################################
+# Helper function get current definitions
+# Usage:
+# mshadow_get_current_definitions(<definitions_variable>)
+function(mshadow_get_current_definitions definitions_var)
+ get_property(current_definitions DIRECTORY PROPERTY COMPILE_DEFINITIONS)
+ set(result "")
+
+ foreach(d ${current_definitions})
+ list(APPEND result -D${d})
+ endforeach()
+
+ mshadow_list_unique(result)
+ set(${definitions_var} ${result} PARENT_SCOPE)
+endfunction()
+
+################################################################################################
+# Helper function get current includes/definitions
+# Usage:
+# mshadow_get_current_cflags(<cflagslist_variable>)
+function(mshadow_get_current_cflags cflags_var)
+ get_property(current_includes DIRECTORY PROPERTY INCLUDE_DIRECTORIES)
+ mshadow_convert_absolute_paths(current_includes)
+ mshadow_get_current_definitions(cflags)
+
+ foreach(i ${current_includes})
+ list(APPEND cflags "-I${i}")
+ endforeach()
+
+ mshadow_list_unique(cflags)
+ set(${cflags_var} ${cflags} PARENT_SCOPE)
+endfunction()
+
+################################################################################################
+# Helper function to parse current linker libs into link directories, libflags and osx frameworks
+# Usage:
+# mshadow_parse_linker_libs(<mshadow_LINKER_LIBS_var> <directories_var> <libflags_var> <frameworks_var>)
+function(mshadow_parse_linker_libs mshadow_LINKER_LIBS_variable folders_var flags_var frameworks_var)
+
+ set(__unspec "")
+ set(__debug "")
+ set(__optimized "")
+ set(__framework "")
+ set(__varname "__unspec")
+
+ # split libs into debug, optimized, unspecified and frameworks
+ foreach(list_elem ${${mshadow_LINKER_LIBS_variable}})
+ if(list_elem STREQUAL "debug")
+ set(__varname "__debug")
+ elseif(list_elem STREQUAL "optimized")
+ set(__varname "__optimized")
+ elseif(list_elem MATCHES "^-framework[ \t]+([^ \t].*)")
+ list(APPEND __framework -framework ${CMAKE_MATCH_1})
+ else()
+ list(APPEND ${__varname} ${list_elem})
+ set(__varname "__unspec")
+ endif()
+ endforeach()
+
+ # attach debug or optimized libs to unspecified according to current configuration
+ if(CMAKE_BUILD_TYPE MATCHES "Debug")
+ set(__libs ${__unspec} ${__debug})
+ else()
+ set(__libs ${__unspec} ${__optimized})
+ endif()
+
+ set(libflags "")
+ set(folders "")
+
+ # convert linker libraries list to link flags
+ foreach(lib ${__libs})
+ if(TARGET ${lib})
+ list(APPEND folders $<TARGET_LINKER_FILE_DIR:${lib}>)
+ list(APPEND libflags -l${lib})
+ elseif(lib MATCHES "^-l.*")
+ list(APPEND libflags ${lib})
+ elseif(IS_ABSOLUTE ${lib})
+ get_filename_component(name_we ${lib} NAME_WE)
+ get_filename_component(folder ${lib} PATH)
+
+ string(REGEX MATCH "^lib(.*)" __match ${name_we})
+ list(APPEND libflags -l${CMAKE_MATCH_1})
+ list(APPEND folders ${folder})
+ else()
+ message(FATAL_ERROR "Logic error. Need to update cmake script")
+ endif()
+ endforeach()
+
+ mshadow_list_unique(libflags folders)
+
+ set(${folders_var} ${folders} PARENT_SCOPE)
+ set(${flags_var} ${libflags} PARENT_SCOPE)
+ set(${frameworks_var} ${__framework} PARENT_SCOPE)
+endfunction()
+
+################################################################################################
+# Helper function to detect Darwin version, i.e. 10.8, 10.9, 10.10, ....
+# Usage:
+# mshadow_detect_darwin_version(<version_variable>)
+function(mshadow_detect_darwin_version output_var)
+ if(APPLE)
+ execute_process(COMMAND /usr/bin/sw_vers -productVersion
+ RESULT_VARIABLE __sw_vers OUTPUT_VARIABLE __sw_vers_out
+ ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+ set(${output_var} ${__sw_vers_out} PARENT_SCOPE)
+ else()
+ set(${output_var} "" PARENT_SCOPE)
+ endif()
+endfunction()
+
+################################################################################################
+# Convenient command to setup source group for IDEs that support this feature (VS, XCode)
+# Usage:
+# caffe_source_group(<group> GLOB[_RECURSE] <globbing_expression>)
+function(mshadow_source_group group)
+ cmake_parse_arguments(CAFFE_SOURCE_GROUP "" "" "GLOB;GLOB_RECURSE" ${ARGN})
+ if(CAFFE_SOURCE_GROUP_GLOB)
+ file(GLOB srcs1 ${CAFFE_SOURCE_GROUP_GLOB})
+ source_group(${group} FILES ${srcs1})
+ endif()
+
+ if(CAFFE_SOURCE_GROUP_GLOB_RECURSE)
+ file(GLOB_RECURSE srcs2 ${CAFFE_SOURCE_GROUP_GLOB_RECURSE})
+ source_group(${group} FILES ${srcs2})
+ endif()
+endfunction()
\ No newline at end of file
diff --git a/3rdparty/mshadow/cmake/mshadow.cmake b/3rdparty/mshadow/cmake/mshadow.cmake
new file mode 100644
index 0000000..1ef7698
--- /dev/null
+++ b/3rdparty/mshadow/cmake/mshadow.cmake
@@ -0,0 +1,91 @@
+set(mshadow_LINKER_LIBS "")
+
+set(BLAS "Open" CACHE STRING "Selected BLAS library")
+set_property(CACHE BLAS PROPERTY STRINGS "Atlas;Open;MKL")
+
+if(DEFINED USE_BLAS)
+ set(BLAS "${USE_BLAS}")
+else()
+ if(USE_MKL_IF_AVAILABLE)
+ if(NOT MKL_FOUND)
+ find_package(MKL)
+ endif()
+ if(MKL_FOUND)
+ set(BLAS "MKL")
+ endif()
+ endif()
+endif()
+
+if(BLAS STREQUAL "Atlas" OR BLAS STREQUAL "atlas")
+ find_package(Atlas REQUIRED)
+ include_directories(SYSTEM ${Atlas_INCLUDE_DIR})
+ list(APPEND mshadow_LINKER_LIBS ${Atlas_LIBRARIES})
+ add_definitions(-DMSHADOW_USE_CBLAS=1)
+ add_definitions(-DMSHADOW_USE_MKL=0)
+elseif(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
+ find_package(OpenBLAS REQUIRED)
+ include_directories(SYSTEM ${OpenBLAS_INCLUDE_DIR})
+ list(APPEND mshadow_LINKER_LIBS ${OpenBLAS_LIB})
+ add_definitions(-DMSHADOW_USE_CBLAS=1)
+ add_definitions(-DMSHADOW_USE_MKL=0)
+elseif(BLAS STREQUAL "MKL" OR BLAS STREQUAL "mkl")
+ find_package(MKL REQUIRED)
+ include_directories(SYSTEM ${MKL_INCLUDE_DIR})
+ list(APPEND mshadow_LINKER_LIBS ${MKL_LIBRARIES})
+ add_definitions(-DMSHADOW_USE_CBLAS=0)
+ add_definitions(-DMSHADOW_USE_MKL=1)
+elseif(BLAS STREQUAL "apple")
+ find_package(Accelerate REQUIRED)
+ include_directories(SYSTEM ${Accelerate_INCLUDE_DIR})
+ list(APPEND mshadow_LINKER_LIBS ${Accelerate_LIBRARIES})
+ add_definitions(-DMSHADOW_USE_MKL=0)
+ add_definitions(-DMSHADOW_USE_CBLAS=1)
+endif()
+
+if(SUPPORT_MSSE2)
+ add_definitions(-DMSHADOW_USE_SSE=1)
+else()
+ add_definitions(-DMSHADOW_USE_SSE=0)
+endif()
+
+if(NOT DEFINED SUPPORT_F16C AND NOT MSVC)
+ check_cxx_compiler_flag("-mf16c" COMPILER_SUPPORT_MF16C)
+ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ execute_process(COMMAND cat /proc/cpuinfo
+ COMMAND grep flags
+ COMMAND grep f16c
+ OUTPUT_VARIABLE CPU_SUPPORT_F16C)
+ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ execute_process(COMMAND sysctl -a
+ COMMAND grep machdep.cpu.features
+ COMMAND grep F16C
+ OUTPUT_VARIABLE CPU_SUPPORT_F16C)
+ endif()
+ if(NOT CPU_SUPPORT_F16C)
+ message("CPU does not support F16C instructions")
+ endif()
+ if(CPU_SUPPORT_F16C AND COMPILER_SUPPORT_MF16C)
+ set(SUPPORT_F16C TRUE)
+ endif()
+endif()
+
+if(SUPPORT_F16C)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mf16c")
+else()
+ add_definitions(-DMSHADOW_USE_F16C=0)
+endif()
+
+if(USE_CUDA)
+ find_package(CUDA 5.5 QUIET)
+ find_cuda_helper_libs(curand)
+ if(NOT CUDA_FOUND)
+ message(FATAL_ERROR "-- CUDA is disabled.")
+ endif()
+ add_definitions(-DMSHADOW_USE_CUDA=1)
+ add_definitions(-DMSHADOW_FORCE_STREAM)
+ include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
+ list(APPEND mshadow_LINKER_LIBS ${CUDA_CUDART_LIBRARY}
+ ${CUDA_curand_LIBRARY} ${CUDA_CUBLAS_LIBRARIES})
+else()
+ add_definitions(-DMSHADOW_USE_CUDA=0)
+endif()
diff --git a/3rdparty/mshadow/cmake/mshadowUtils.cmake b/3rdparty/mshadow/cmake/mshadowUtils.cmake
new file mode 100644
index 0000000..d4b8bfc
--- /dev/null
+++ b/3rdparty/mshadow/cmake/mshadowUtils.cmake
@@ -0,0 +1,2 @@
+include("${CMAKE_CURRENT_LIST_DIR}/Utils.cmake")
+
diff --git a/3rdparty/mshadow/doc/Doxyfile b/3rdparty/mshadow/doc/Doxyfile
new file mode 100644
index 0000000..3e83d47
--- /dev/null
+++ b/3rdparty/mshadow/doc/Doxyfile
@@ -0,0 +1,2358 @@
+# Doxyfile 1.8.8
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME = "mshadow"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER =
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF =
+
+# With the PROJECT_LOGO tag one can specify an logo or icon that is included in
+# the documentation. The maximum height of the logo should not exceed 55 pixels
+# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo
+# to the output directory.
+
+PROJECT_LOGO =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY = doc
+
+# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC = YES
+
+# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB = NO
+
+# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES = YES
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a
+# new page for each member. If set to NO, the documentation of a member will be
+# part of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE = 8
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C = YES
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
+# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
+# Fortran. In the later case the parser tries to guess whether the code is fixed
+# or free formatted code, this is the default for Fortran type files), VHDL. For
+# instance to make doxygen treat .inc files as Fortran files (default is PHP),
+# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+#
+# Note For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by by putting a % sign in front of the word
+# or globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES, then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL = NO
+
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE = NO
+
+# If the EXTRACT_STATIC tag is set to YES all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES = YES
+
+# This flag is only useful for Objective-C code. When set to YES local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO these classes will be included in the various overviews. This option has
+# no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES = YES
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES = NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING = NO
+
+# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the
+# todo list. This list is created by putting \todo commands in the
+# documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST = YES
+
+# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the
+# test list. This list is created by putting \test commands in the
+# documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST = YES
+
+# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES the list
+# will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO doxygen will only warn about wrong or incomplete parameter
+# documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC = YES
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces.
+# Note: If this tag is empty the current directory is searched.
+
+INPUT = mshadow \
+ mshadow-ps
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank the
+# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii,
+# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp,
+# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown,
+# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
+# *.qsf, *.as and *.js.
+
+FILE_PATTERNS =
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE = NO
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS = *-inl.* \
+ utils.h \
+ thread_util.h \
+ thread.h \
+ kv_array.h
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS = mshadow::expr::Plan* \
+ mshadow::expr::*Engine*
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+
+INPUT_FILTER =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+
+FILTER_PATTERNS =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER ) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = NO
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION = NO
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES, then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS = YES
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES, then doxygen will use the
+# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
+# cost of reduced performance. This can be particularly helpful with template
+# rich C++ code for which doxygen's built-in parser lacks the necessary type
+# information.
+# Note: The availability of this option depends on whether or not doxygen was
+# compiled with the --with-libclang option.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefor more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra stylesheet files is of importance (e.g. the last
+# stylesheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the stylesheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP = YES
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler ( hhc.exe). If non-empty
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated (
+# YES) or that it should be included in the master .chm file ( NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated (
+# YES) or a normal table of contents ( NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH = 250
+
+# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE = 10
+
+# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using prerendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH = http://www.mathjax.org/mathjax
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer ( doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH = NO
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer ( doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX = YES
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME = makeindex
+
+# If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. To get the times font for
+# instance you can specify
+# EXTRA_PACKAGES=times
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empy string,
+# for the replacement values of the other commands the user is refered to
+# HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE = plain
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT = rtf
+
+# If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen
+# Definitions (see http://autogen.sf.net) file that captures the structure of
+# the code including all documentation. Note that this feature is still
+# experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD = NO
+
+# If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX = NO
+
+# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING = NO
+
+# If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names
+# in the source code. If set to NO only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF = NO
+
+# If the SEARCH_INCLUDES tag is set to YES the includes files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE =
+
+# If the ALLEXTERNALS tag is set to YES all external class will be listed in the
+# class index. If set to NO only the inherited external classes will be listed.
+# The default value is: NO.
+
+ALLEXTERNALS = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in
+# the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS = YES
+
+# If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH =
+
+# If set to YES, the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: YES.
+
+HAVE_DOT = NO
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS = YES
+
+# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot.
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
+# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
+# gif:cairo:gd, gif:gd, gif:gd:gd and svg.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+PLANTUML_JAR_PATH =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS = YES
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND = YES
+
+# If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP = YES
diff --git a/3rdparty/mshadow/doc/README.md b/3rdparty/mshadow/doc/README.md
new file mode 100644
index 0000000..2fed344
--- /dev/null
+++ b/3rdparty/mshadow/doc/README.md
@@ -0,0 +1,321 @@
+MShadow Documentation
+=====
+This is the documentation for mshadow: A Lightweight CPU/GPU Matrix/Tensor Template Library in C++/CUDA.
+
+### Links to Topics
+
+* [Tutorial](../guide)
+* [API Documentation](http://homes.cs.washington.edu/~tqchen/mshadow/doc)
+ - You can run ```./mkdoc.sh``` to make the document locally
+* [Tutorial about Expression Template](../guide/exp-template)
+* [Writing Multi-GPU and Distributed ML](../guide/mshadow-ps)
+* [Compile Configuration script](../make)
+* [Expression API](#expression-api)
+ - Expression api introduces the concept of expression in mshadow
+
+Expression API
+=====
+Expression is the key concept in mshadow, a common operation of mshadow is ```tensor = some code to construct expression```
+
+There are three major types of expression:
+* Mapper expression: only contain element-wise operations of Mapper expressions
+ - Mapper expression can used as composition component of other operations.
+ - Tensor, scalar are Mapper expressions
+ - Example: ``` weight = - eta * (grad + lambda * weight)``` is a Mapper expression.
+ - Mapper expressions are translated using expression template code implemented by mshadow.
+ - ***Assign safety***: Element-wise mapping are assign safe, which means, we can write ```A = A * 2 + B```, making lvalue appear in expression, the results are still correct.
+* Chainer expression: may contain element-wise operation such as reduction and broadcast
+ - Example: ```dst = mirror(src)``` is a chainer expression
+ - ***Assign safety***: Most of the chainer extensions are not assignment safe, which means user should avoid putting target in source epression.
+* Complex expression: complex operations, need special translation rule to translate to specific implementations.
+ - Complex expression can not be used as composition component of other operations.
+ - Example: ``` dot(lhs.T(), rhs)```, is complex expression, we can not write
+``` dst = 1.0 + dot(lhs.T(), rhs)```
+ - But limited syntax is supported depending on specification, for example, we do support ``` dst += 2.0f * dot(lhs.T(), rhs)```
+ - Complex expressions are translated into specific implementations such as BLAS.
+
+### Element-wise Operations
+The basic binary operators are overloaded to composite Mapper expressions, so we can write
+```c++
+weight = (-eta) * (grad + lambda * weight);
+```
+We can also use customized binary operators, and unary operators:
+```c++
+struct maximum {
+ MSHADOW_XINLINE static float Map(float a, float b) {
+ return a > b ? a : b;
+ }
+};
+template<typename xpu>
+void ExampleMaximum(Tensor<xpu, 2> out,
+ const Tensor<xpu, 2> &A,
+ const Tensor<xpu, 2> &B) {
+ out= 10.0f * F<maximum>(A+1.0f, B);
+}
+struct sigmoid {
+ MSHADOW_XINLINE static float Map(float a) {
+ return 1.0f/(1.0f+expf(-a));
+ }
+};
+template<typename xpu>
+void ExampleSigmoid(Tensor<xpu, 2> out, const Tensor<xpu, 2> &in) {
+ // equivalent to out = sigmoid(in*2) + 1;
+ out = F<op::plus>(F<sigmoid>(in * 2.0f), ScalarExp(1.0f));
+}
+```
+### Matrix Multiplications
+Matrix multiplications are supported by following syntax, with things brackets [] are optional
+```
+dst <sv> [scale*] dot(lhs [.T()] , rhs [.T()]), <sv> can be =,+=,-=
+```
+Example:
+```c++
+template<typename xpu>
+void Backprop(Tensor<xpu, 2> gradin,
+ const Tensor<xpu, 2> &gradout,
+ const Tensor<xpu, 2> &netweight) {
+ gradin = 2.0 * dot(gradout, netweight.T());
+}
+```
+
+### Introducing Expression Extensions
+Naming conventions:
+* ```Tensor<xpu, dim>``` to refer to any Tensor with device any device and dimension.
+* ```xpu```, ```dim```, are implicit template parameters.
+* ```Expr<xpu, dim>``` will be used to refer to any mapper expression with type ```Tensor<xpu,dim>```.
+
+List of functions:
+* [reshape](#reshape): reshapes a tensor to another shape, number of content must be same
+* [broadcast<?>](#broadcast): replicate a 1 dimension tensor in certain dimension
+* [repmat](#repmat), special case of broadcast<0>: repeat vector over rows to form a matrix
+* [sumall_except_dim<?>](#sumall_except_dim): sum over all the dimensions, except the dimension specified in template parameter
+* [sum_rows](#sum_rows): special case of sumall_except_dim<0>, sum of rows in the matrix
+* [unpack_patch2col](#unpack_patch2col): unpack local (overlap) patches of image to column of mat, can be used to implement convolution
+* [pack_col2patch](#pack_col2patch): reverse operation of unpack_patch2col, can be used to implement deconvolution
+* [pool](#pool): do pooling on image
+* [unpool](#unpool): get gradient of pooling result
+* [crop](#crop): crop the original image to a smaller size
+* [mirror](#mirror): get the mirrored result of input expression
+
+======
+##### reshape
+* ```reshape(Expr<xpu,dim> src, Shape<dimdst> oshape)```
+* reshapes a tensor to another shape, total number of elements must be same
+* parameters:
+ - src: input data
+ - oshape: target shape
+* result expression type: ```Tensor<xpu, dimdst>``` with ```shape=oshape```, is Mapper expression
+```c++
+void ExampleReshape(void) {
+ Tensor<cpu, 2> dst = NewTensor<cpu>(Shape2(4, 5));
+ Tensor<cpu, 1> src = NewTensor<cpu>(Shape1(20), 1.0f);
+ dst = reshape(src, dst.shape_);
+ ...
+}
+```
+======
+
+##### broadcast
+* ```broadcast<dimcast>(Tensor<xpu,1> src, Shape<dimdst> oshape)```
+* replicate a 1 dimension tensor certain dimension, specified by template parameter dimcast
+* parameters:
+ - src: input 1 dimensional tensor
+ - oshape: shape of output
+* return expression type: ```Tensor<xpu, dimdst>```, ```shape = oshape```, is Chainer expression
+```c++
+void ExampleBroadcast(void) {
+ Tensor<cpu, 2> dst = NewTensor<cpu>(Shape2(2, 3));
+ Tensor<cpu, 1> src = NewTensor<cpu>(Shape1(2), 1.0f);
+ src[0] = 2.0f; src[1] = 1.0f;
+ dst = broadcast<0>(src, dst.shape_);
+ // dst[0][0] = 2, dst[0][1] = 2; dst[1][0]=1, dst[1][1] = 1
+ ...
+}
+```
+======
+##### repmat
+* ```repmat(Tensor<xpu, 1> src, int nrows) ```
+* special case of broadcast, repeat 1d tensor over rows
+* input parameters:
+ - src: input vector
+ - nrows: number of rows in target
+* return expression type: ```Tensor<xpu, 2>```, with ```shape=(nrows, src.size(0))```, is Chainer expression
+```c++
+void ExampleRepmat(void) {
+ Tensor<cpu,2> dst = NewTensor<cpu>(Shape2(3, 2));
+ Tensor<cpu,1> src = NewTensor<cpu>(Shape1(2), 1.0f);
+ src[0] = 2.0f; src[1] = 1.0f;
+ dst = repmat(src, 3);
+ // dst[0][0] = 2, dst[0][1] = 1; dst[1][0]=2, dst[1][1] = 1
+ ...
+}
+```
+======
+##### sumall_except_dim
+* ```sumall_except_dim<dimkeep>(Expr<xpu,dim> src) ```
+* sum over all dimensions, except dimkeep
+* input parameters:
+ - src: input mapper expression
+* return expression type: ```Tensor<xpu, 1>```, with ```shape=(src.size(dimkeep))```, is Complex expression
+* Syntax: ```dst [sv] [scale*] sumall_except_dim<dimkeep>(src) , <sv> can be =, +=, -=, *=, /=````
+```c++
+void ExampleSumAllExceptDim(void) {
+ Tensor<cpu,3> src = NewTensor<cpu>(Shape3(2, 3, 2), 1.0f);
+ Tensor<cpu,1> dst = NewTensor<cpu>(Shape1(3), 1.0f);
+ dst += sum_all_except<1>(src * 2.0f);
+ // dst[0] = 1.0 + 4.0 *2.0 = 9.0
+ ...
+}
+```
+======
+##### sum_rows
+* ```sum_rows(Expr<xpu, 2> src) ```
+* sum of rows in the matrix
+* input parameters:
+ - src: input mapper expression
+* return expression type: ```Tensor<xpu,1>```, with ```shape=(src.size(0))```, is Complex expression
+* Syntax: ```dst [sv] [scale*] sum_rows(src) , <sv> can be =,+=,-=,*=,/=````
+```c++
+void ExampleSumRows(void) {
+ Tensor<cpu, 2> src = NewTensor<cpu>(Shape2(3, 2), 1.0f);
+ Tensor<cpu, 1> dst = NewTensor<cpu>(Shape1(2), 1.0f);
+ dst += sum_rows(src + 1.0f);
+ // dst[0] = 1.0 + 3.0 *(1.0+1.0) = 7.0
+ ...
+}
+```
+======
+##### unpack_patch2col
+* ```unpack_patch2col(Expr<xpu,3> img, int psize_y, int p_size_x, int pstride) ```
+* unpack local (overlap) patches of image to column of mat, can be used to implement convolution, after getting unpacked mat, we can use: ```output = dot(weight, mat)``` to get covolved results, the relations:
+ - weight; shape[0]: out_channel, shape[1]: ichannel * psize_y * psize_x
+ - output; shape[0]: out_channel, shape[1]: out_height * out_width * num_of_images
+ - out_height = (in_height - psize_y) / pstride + 1, this means we pad inperfect patch with 0
+ - out_width = (in_width - psize_x) / pstride + 1
+* input parameters:
+ - img: source image, can be expression; (in_channels, in_height, in_width)
+ - psize_y height of each patch
+ - psize_x width of each patch
+ - pstride: stride of each patch
+* return expression type: ```Tensor<xpu, 2>```, with ```shape=(in_channel*psize_x*psize_y, out_height*out_width)```, is Chainer expression
+```c++
+void ExampleCovolution(Tensor<cpu, 3> dst, Tensor<cpu, 3> src,
+ Tensor<cpu, 2> weight, int ksize, int stride) {
+ int o_height = (src.size(1)- ksize) / stride + 1;
+ int o_width = (src.size(2)- ksize) / stride + 1;
+ utils::Assert(weight.size(1) == src.size(0) * ksize * ksize);
+ TensorContainer<cpu, 2> tmp_col(Shape2(src.size(0) * ksize * ksize,
+ o_height * o_width));
+ TensorContainer<cpu, 2> tmp_dst(Shape2(weight.size(0),
+ o_height * o_width));
+ tmp_col = unpack_patch2col(src, ksize, ksize, stride);
+ tmp_dst = dot(weight, tmp_col);
+ dst = reshape(tmp_dst, dst.shape_);
+}
+```
+
+======
+##### pack_col2patch
+* ```pack_col2patch(Tensor<xpu, 2> mat, Shape<3> imshape, int psize_y, int psize_x, int pstride) ````
+* reverse operation of unpack_patch2col, can be used to implement deconvolution
+* input parameters:
+ - mat: source mat, same shape as output of unpack_patch2col
+ - imshape: shape of target image
+ - psize_y height of each patch
+ - psize_x width of each patch
+ - pstride: stride of each patch
+* return expression type: ```Tensor<xpu, 3>```, with ```shape = imshape```, is Chainer expression
+```c++
+void ExampleDecovolution(Tensor<cpu, 3> bottom, Tensor<cpu, 3> top,
+ Tensor<cpu, 2> weight, int ksize, int stride) {
+ int o_height = (bottom.size(1)- ksize) / stride + 1;
+ int o_width = (bottom.size(2)- ksize) / stride + 1;
+ utils::Assert(weight.size(1) == bottom.size(0) * ksize * ksize);
+ TensorContainer<cpu, 2> tmp_col(Shape2(bottom.size(0) * ksize * ksize,
+ o_height * o_width));
+ TensorContainer<cpu, 2> tmp_dst(Shape2(weight.size(0), o_height*o_width));
+ tmp_dst = reshape(top, tmp_dst.shape_);
+ tmp_col = dot(weight.T(), tmp_dst);
+ bottom = pack_col2patch(tmp_col, bottom.shape_, ksize, ksize, stride);
+}
+```
+
+======
+##### pool
+* ```pool<Reducer>(Expr<xpu, dim> img, [Shape<2> pshape,] int ksize_y, int ksize_x, int kstride)```
+* Pooling on image with specify kernel size and stride, can be used to implement max pooilng and other pooling layer
+* input parameters:
+ - Reducer: operation can be max or sum
+ - img: source image, can be expression; (in_channels, in_height, in_width)
+ - [optional] Shape<2> pshape, output shape
+ - ksize_y height of each patch
+ - ksize_x width of each patch
+ - kstride: stride of each patch
+* return expression: ```Expr<xpu, dim>```, with ```shape = (in_channel, (out_height - ksize) / kstride + 1, (out_width - ksize) / kstride + 1)```, or expression in pshape
+ - Chainer expression
+```c++
+void ExampleMaxPooling(TensorContainer<cpu, 3> &data, int ksize, int stride) {
+ TensorContainer<cpu, 3> pooled(Shape3(data.size(0),
+ (data.size(2) - ksize) / kstride + 1),
+ (data.size(1) - ksize) / kstride + 1));
+ pooled = pool<red::maximum>(data, ksize, ksize, stride);
+}
+```
+
+======
+##### unpool
+* ```unpool<Reducer>(Tensor<xpu, 4> data_src, Tensor<xpu, 4> data_pooled, Tensor<xpu, 4> grad_pooled, int ksize_y, int ksize_x, int kstride)```
+* Unpooling on image with specify kernel size and stride, can be used to implement backprop of max pooilng and other pooling layer
+* input parameters:
+ - Reducer: operation can be max or sum
+ - data_src: source image batch.
+ - data_pooled: pooled image batch.
+ - grad_pooled: gradient of upper layer
+ - ksize_y height of each patch
+ - ksize_x width of each patch
+ - kstride: stride of each patch
+* return:
+ Expression, same shape to data_src
+```c++
+void ExampleMaxUnpooling(Tensor<cpu, 4> &data_src, Tensor<cpu, 4> &data_pooled,
+ Tensor<cpu, 4> &grad_pooled, int ksize, int kstride) {
+ TensorContainer<cpu, 4> grad(data_src.shape_);
+ grad = unpool<red::maximum>(data_src, data_pooled,
+ grad_pooled, ksize, ksize, kstride);
+}
+```
+
+======
+##### crop
+* ```crop(Expr<xpu, dim> src, Shape<2> oshape, int start_height, int start_width)```
+* input parameters:
+ - src: input expression
+ - oshape: output shape after crop
+ - start_height: start height for cropping
+ - start_width: start width for cropping
+* Can also be ```crop(Expr<xpu, dim> src, Shape<2> oshape)``` where the crop will happen in center.
+* return
+ - cropped expression
+```c++
+void ExampleCrop(TensorContainer<cpu, 3> img, int start_height, int start_width) {
+ TensorContainer<cpu> cropped(Shape3(img.size(0),
+ img.size(1) - start_height,
+ img.size(2) - start_width));
+ cropped = crop(img, start_height, start_width);
+}
+```
+
+======
+##### mirror
+* ```mirrow(Expr<xpu, dim> src)```
+* input:
+ - src, source expression to be mirrored
+* output:
+ - expression of mirrored result
+```c++
+void ExampleMirror(TensorContainer<cpu, 3> img) {
+ TensorContainer<cpu> mirrored(img.shape_);
+ mirrored = mirror(img);
+}
+```
+
diff --git a/3rdparty/mshadow/doc/mkdoc.sh b/3rdparty/mshadow/doc/mkdoc.sh
new file mode 100755
index 0000000..3ee3d71
--- /dev/null
+++ b/3rdparty/mshadow/doc/mkdoc.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+cd ..
+doxygen doc/Doxyfile
+cd doc
diff --git a/3rdparty/mshadow/guide/.gitignore b/3rdparty/mshadow/guide/.gitignore
new file mode 100644
index 0000000..ef77967
--- /dev/null
+++ b/3rdparty/mshadow/guide/.gitignore
@@ -0,0 +1,3 @@
+defop
+basic
+config.mk
diff --git a/3rdparty/mshadow/guide/Makefile b/3rdparty/mshadow/guide/Makefile
new file mode 100644
index 0000000..bad7a8e
--- /dev/null
+++ b/3rdparty/mshadow/guide/Makefile
@@ -0,0 +1,37 @@
+# set LD_LIBRARY_PATH
+export CC = gcc
+export CXX = g++
+export NVCC =nvcc
+include config.mk
+include ../make/mshadow.mk
+export CFLAGS = -Wall -O3 -std=c++11 -I../ $(MSHADOW_CFLAGS)
+export LDFLAGS= -lm $(MSHADOW_LDFLAGS)
+export NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
+
+# specify tensor path
+BIN = basic defop
+OBJ =
+CUOBJ =
+CUBIN =
+.PHONY: clean all
+
+all: $(BIN) $(OBJ) $(CUBIN) $(CUOBJ)
+
+basic: basic.cpp
+defop: defop.cpp
+basic_stream: basic_stream.cu
+
+$(BIN) :
+ $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^) $(LDFLAGS)
+
+$(OBJ) :
+ $(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
+
+$(CUOBJ) :
+ $(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" $(filter %.cu, $^)
+
+$(CUBIN) :
+ $(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" -Xlinker "$(LDFLAGS)" $(filter %.cu %.cpp %.o, $^)
+
+clean:
+ $(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~
diff --git a/3rdparty/mshadow/guide/README.md b/3rdparty/mshadow/guide/README.md
new file mode 100644
index 0000000..3f2e681
--- /dev/null
+++ b/3rdparty/mshadow/guide/README.md
@@ -0,0 +1,226 @@
+Tutorial of mshadow
+=====
+This is a beginner's tutorial for mshadow. If you like mshadow and have ideas to improve this tutorial, you are more than welcome to contribute :)
+Please send a pull-request if you would like to share your experience.
+
+See also other related materials about mshadow
+* [Expression Template Tutorial](exp-template)
+* [Writing Multi-GPU and Distributed ML](mshadow-ps)
+
+**List of Topics**
+* [Tensor Data Structure](#tensor-data-structure)
+* [Memory Allocation](#memory-allocation)
+* [Elementwise Operations](#elementwise-operations)
+* [One code for both CPU and GPU](#one-code-for-both-cpu-and-gpu)
+* [Matrix Multiplications](#matrix-multiplications)
+* [User Defined Operator](#user-defined-operator)
+
+Tensor Data Structure
+====
+The basic data structure of mshadow is Tensor. The following is a simplified equivalent version of
+the declaration in [mashadow/tensor.h](../mshadow/tensor.h)
+```c++
+typedef unsigned index_t;
+template<int dimension>
+struct Shape {
+ index_t shape_[dimension];
+};
+template<typename Device, int dimension, typename DType = float>
+struct Tensor {
+ DType *dptr_;
+ Shape<dimension> shape_;
+ Stream<Device> stream_;
+ index_t stride_;
+};
+// this is how shape object declaration look like
+Shape<2> shape2;
+// this is how tensor object declaration look like
+// you can
+Tensor<cpu, 2> ts2;
+Tensor<gpu, 3, float> ts3;
+```
+``` Tensor<cpu,2>``` is a two dimensional tensor in host memory, while ```Tensor<gpu,3>``` is a three dimensional tensor in device memory.
+```Shape<k>``` gives the shape information of a k-dimensional tensor. The declarations use templates and
+can be specialized to tensors on a specific device and of a specific dimension. This is what a two dimensional tensor would look like:
+```c++
+struct Shape<2> {
+ index_t shape_[2];
+};
+struct Tensor<cpu, 2, float> {
+ float *dptr_;
+ Shape<2> shape_;
+ index_t stride_;
+};
+```
+* ``` Tensor<cpu, 2>``` contains ```dptr_```, which points to the space that backs up the tensor.
+* ```Shape<2>``` is a structure that stores shape information, the convention is the same as numpy.
+* ```stride_``` gives the number of cell spaces allocated in the smallest dimension (if we use numpy convention, the dimension corresponds to shape_[-1]).
+This is introduced when we introduce some padding cells in lowest dimension to make sure memory is aligned. ```stride_``` is automatically set during
+memory allocation of a tensor in mshadow.
+
+To understand the data structure, consider the following code:
+``` c++
+float data[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8};
+Tensor<cpu, 2> ts;
+ts.dptr_ = data;
+ts.shape_ = mshadow::Shape2(3, 2);
+ts.stride_ = 3;
+// now: ts[0][0] == 0, ts[0][1] == 1 , ts[1][0] == 3, ts[1][1] == 4
+for (index_t i = 0; i < ts.size(0); ++i) {
+ for (index_t j = 0; j < ts.size(1); ++j) {
+ printf("ts[%u][%u]=%f\n", i, j, ts[i][j]);
+ }
+}
+```
+The result ts should be a 3 * 2 matrix, where data[2], data[5], data[8] are padding cells that are ignored. If you want a continuous memory, set ```stride_=shape_[1]```.
+
+NOTICE: We highly recommend use stream in ```gpu``` mode, there will be an error thrown out if no stream is set. Check [basic_stream.cu](basic_stream.cu) for more detail.
+
+Memory Allocation
+====
+An important design choice in mshadow was making the data structure ```Tensor``` a **whitebox**,
+it works so long as we set the space pointer ```dptr_``` corresponding ```shape_``` and ```stride_```:
+* For ```Tensor<cpu, k>``` ```dptr_``` must point to space created by ```new float[]``` or to some existing space such as the float array in the last example.
+* For ```Tensor<gpu, k>``` ```dptr_``` must point to space on the device created by ```cudaMallocPitch```.
+
+mshadow also provides an explicit memory allocation routine, as shown in following code:
+``` c++
+// create a 5 x 3 tensor on the device, and allocate space
+Tensor<gpu, 2> ts2(Shape2(5, 3));
+AllocSpace(&ts2);
+// allocate 5 x 3 x 2 tensor on the host, initialized by 0
+Tensor<cpu, 3> ts3 = NewTensor<cpu>(Shape3(5,3,2), 0.0f);
+// free space
+FreeSpace(&ts2); FreeSpace(&ts3);
+```
+All memory allocations in mshadow are **explicit**. There are **no** implicit memory allocations or de-allocations during any operations.
+This means ```Tensor<cpu, k>``` variable is more like a reference handle(pointer), instead of a object. If we assign a tensor to another variable, the two share the same content space.
+
+This also allows user to use mshadow in their existing project easily, simply give mshadow the pointer of the memory and you can get the benefit of all the mshadow expressions with zero cost:)
+
+We also have STL style container object called ```TensorContainer```, they behave exactly the same as Tensors, but the memory will be automatically freed during destruction.
+
+Elementwise Operations
+====
+All the operators(+, -, *, /, += etc.) in mshadow are element-wise. Consider the following SGD update code:
+```c++
+void UpdateSGD(Tensor<cpu, 2> weight, Tensor<cpu, 2> grad, float eta, float lambda) {
+ weight -= eta * (grad + lambda * weight);
+}
+```
+During compilation, this code will be translated to the following form:
+```c++
+void UpdateSGD(Tensor<cpu,2> weight, Tensor<cpu,2> grad, float eta, float lambda) {
+ for (index_t y = 0; y < weight.size(0); ++y) {
+ for (index_t x = 0; x < weight.size(1); ++x) {
+ weight[y][x] -= eta * (grad[y][x] + lambda * weight[y][x]);
+ }
+ }
+}
+```
+As we can see, *no memory allocation* happens in the translated code. For ```Tensor<gpu, k>```, the corresponding function will be translated into a CUDA kernel of the same spirit.
+Using an [Expression Template](exp-template), the translation happens at compile time. We can write simple lines of code while getting the full performance of the translated code.
+
+One code for both CPU and GPU
+====
+Since mshadow has an identical interface for ```Tensor<cpu, k>``` and ```Tensor<gpu, k>```, we can easily write code that works on both the CPU and GPU.
+For example, the following code compiles for both GPU and CPU Tensors.
+```c++
+template<typename xpu>
+void UpdateSGD(Tensor<xpu, 2> weight, const Tensor<xpu, 2> &grad,
+ float eta, float lambda) {
+ weight -= eta * (grad + lambda * weight);
+}
+```
+Matrix Multiplications
+====
+We also have a shorthand for dot product that will be translated to call standard packages such as MKL and CuBLAS.
+```c++
+template<typename xpu>
+void Backprop(Tensor<xpu, 2> gradin,
+ const Tensor<xpu, 2> &gradout,
+ const Tensor<xpu, 2> &netweight) {
+ gradin = dot(gradout, netweight.T());
+}
+```
+Again, the code can compile for both GPU and CPU Tensors.
+
+User Defined Operator
+====
+There are common cases when we want to define our own function. For example, assume we do not have an element-wise sigmoid transformation in mshadow.
+We simply use the following code to add ```sigmoid``` to mshadow
+```c++
+struct sigmoid {
+ MSHADOW_XINLINE static float Map(float a) {
+ return 1.0f / (1.0f + expf(-a));
+ }
+};
+template<typename xpu>
+void ExampleSigmoid(Tensor<xpu, 2> out, const Tensor<xpu, 2> &in) {
+ out = F<sigmoid>(in * 2.0f) + 1.0f;
+}
+```
+The translated code for CPU is given by
+```c++
+template<typename xpu>
+void ExampleSigmoid(Tensor<xpu, 2> out, const Tensor<xpu, 2> &in) {
+ for (index_t y = 0; y < out.size(0); ++y) {
+ for(index_t x = 0; x < out.size(1); ++x) {
+ out[y][x] = sigmoid::Map(in[y][x] * 2.0f) + 1.0f;
+ }
+ }
+}
+```
+Also note that the defined operation can be **composited into expressions**, not only we can write ```out = F<sigmoid>(in)```,
+we can also write ```out = F<sigmoid>+2.0``` or ```out = F<sigmoid>(F<sigmoid>(in))```.
+
+There will also be a translated CUDA kernel version that runs on the GPU. Check out [defop.cpp](defop.cpp) for a complete example.
+
+Complete Example
+====
+The following code is from [basic.cpp](basic.cpp). It illustrates basic usage of mshadow.
+
+```c++
+// header file to use mshadow
+#include "mshadow/tensor.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+int main(void) {
+ // intialize tensor engine before using tensor operation, needed for CuBLAS
+ InitTensorEngine<cpu>();
+ // assume we have a float space
+ float data[20];
+ // create a 2 x 5 x 2 tensor, from existing space
+ Tensor<cpu, 3> ts(data, Shape3(2,5,2));
+ // take first subscript of the tensor
+ Tensor<cpu, 2> mat = ts[0];
+ // Tensor object is only a handle, assignment means they have same data content
+ // we can specify content type of a Tensor, if not specified, it is float bydefault
+ Tensor<cpu, 2, float> mat2 = mat;
+
+ // shape of matrix, note size order is the same as numpy
+ printf("%u X %u matrix\n", mat.size(0), mat.size(1));
+
+ // initialize all element to zero
+ mat = 0.0f;
+ // assign some values
+ mat[0][1] = 1.0f; mat[1][0] = 2.0f;
+ // elementwise operations
+ mat += (mat + 10.0f) / 10.0f + 2.0f;
+
+ // print out matrix, note: mat2 and mat1 are handles(pointers)
+ for (index_t i = 0; i < mat.size(0); ++i) {
+ for (index_t j = 0; j < mat.size(1); ++j) {
+ printf("%.2f ", mat2[i][j]);
+ }
+ printf("\n");
+ }
+ // shutdown tensor enigne after usage
+ ShutdownTensorEngine<cpu>();
+ return 0;
+}
+```
+
diff --git a/3rdparty/mshadow/guide/basic.cpp b/3rdparty/mshadow/guide/basic.cpp
new file mode 100644
index 0000000..8e90ac9
--- /dev/null
+++ b/3rdparty/mshadow/guide/basic.cpp
@@ -0,0 +1,161 @@
+// header file to use mshadow
+#include "mshadow/tensor.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+int main(void) {
+ // intialize tensor engine before using tensor operation, needed for CuBLAS
+ InitTensorEngine<cpu>();
+ // assume we have a float space
+ float data[20];
+ // create a 2 x 5 x 2 tensor, from existing space
+ Tensor<cpu, 3> ts(data, Shape3(2,5,2));
+ // take first subscript of the tensor
+ Tensor<cpu, 2> mat = ts[0];
+ // Tensor object is only a handle, assignment means they have same data content
+ // we can specify content type of a Tensor, if not specified, it is float bydefault
+ Tensor<cpu, 2, float> mat2 = mat;
+ mat = Tensor<cpu, 1>(data, Shape1(10)).FlatTo2D();
+
+ // shaape of matrix, note size order is same as numpy
+ printf("%u X %u matrix\n", mat.size(0), mat.size(1));
+
+ // initialize all element to zero
+ mat = 0.0f;
+ // assign some values
+ mat[0][1] = 1.0f; mat[1][0] = 2.0f;
+ // elementwise operations
+ mat += (mat + 10.0f) / 10.0f + 2.0f;
+
+ // print out matrix, note: mat2 and mat1 are handles(pointers)
+ for (index_t i = 0; i < mat.size(0); ++i) {
+ for (index_t j = 0; j < mat.size(1); ++j) {
+ printf("%.2f ", mat2[i][j]);
+ }
+ printf("\n");
+ }
+
+ TensorContainer<cpu, 2> lhs(Shape2(2, 3)), rhs(Shape2(2, 3)), ret(Shape2(2,2));
+ lhs = 1.0;
+ rhs = 1.0;
+ ret = implicit_dot(lhs, rhs.T());
+ VectorDot(ret[0].Slice(0, 1), lhs[0], rhs[0]);
+ printf("vdot=%f\n", ret[0][0]);
+ int cnt = 0;
+ for (index_t i = 0; i < ret.size(0); ++i) {
+ for (index_t j = 0; j < ret.size(1); ++j) {
+ printf("%.2f ", ret[i][j]);
+ }
+ printf("\n");
+ }
+
+ printf("\n");
+
+ for (index_t i = 0; i < lhs.size(0); ++i) {
+ for (index_t j = 0; j < lhs.size(1); ++j) {
+ lhs[i][j] = cnt++;
+ printf("%.2f ", lhs[i][j]);
+ }
+ printf("\n");
+ }
+ printf("\n");
+ TensorContainer<cpu, 1> index(Shape1(2)), choosed(Shape1(2));
+ index[0] = 1; index[1] = 2;
+ choosed = mat_choose_row_element(lhs, index);
+ for (index_t i = 0; i < choosed.size(0); ++i) {
+ printf("%.2f ", choosed[i]);
+ }
+ printf("\n");
+
+ TensorContainer<cpu, 2> recover_lhs(Shape2(2, 3)), small_mat(Shape2(2, 3));
+ small_mat = -100.0f;
+ recover_lhs = mat_fill_row_element(small_mat, choosed, index);
+ for (index_t i = 0; i < recover_lhs.size(0); ++i) {
+ for (index_t j = 0; j < recover_lhs.size(1); ++j) {
+ printf("%.2f ", recover_lhs[i][j] - lhs[i][j]);
+ }
+ }
+ printf("\n");
+
+ rhs = one_hot_encode(index, 3);
+
+ for (index_t i = 0; i < lhs.size(0); ++i) {
+ for (index_t j = 0; j < lhs.size(1); ++j) {
+ printf("%.2f ", rhs[i][j]);
+ }
+ printf("\n");
+ }
+ printf("\n");
+ TensorContainer<cpu, 1> idx(Shape1(3));
+ idx[0] = 8;
+ idx[1] = 0;
+ idx[2] = 1;
+
+ TensorContainer<cpu, 2> weight(Shape2(10, 5));
+ TensorContainer<cpu, 2> embed(Shape2(3, 5));
+
+ for (index_t i = 0; i < weight.size(0); ++i) {
+ for (index_t j = 0; j < weight.size(1); ++j) {
+ weight[i][j] = i;
+ }
+ }
+ embed = take(idx, weight);
+ for (index_t i = 0; i < embed.size(0); ++i) {
+ for (index_t j = 0; j < embed.size(1); ++j) {
+ printf("%.2f ", embed[i][j]);
+ }
+ printf("\n");
+ }
+ printf("\n\n");
+ weight = take_grad(idx, embed, 10);
+ for (index_t i = 0; i < weight.size(0); ++i) {
+ for (index_t j = 0; j < weight.size(1); ++j) {
+ printf("%.2f ", weight[i][j]);
+ }
+ printf("\n");
+ }
+
+ printf("upsampling\n");
+ TensorContainer<cpu, 2> small(Shape2(2, 2));
+ small[0][0] = 1.0f;
+ small[0][1] = 2.0f;
+ small[1][0] = 3.0f;
+ small[1][1] = 4.0f;
+ TensorContainer<cpu, 2> large(Shape2(6, 6));
+ large = upsampling_nearest(small, 3);
+ for (index_t i = 0; i < large.size(0); ++i) {
+ for (index_t j = 0; j < large.size(1); ++j) {
+ printf("%.2f ", large[i][j]);
+ }
+ printf("\n");
+ }
+ small = pool<red::sum>(large, small.shape_, 3, 3, 3, 3);
+ // shutdown tensor enigne after usage
+ for (index_t i = 0; i < small.size(0); ++i) {
+ for (index_t j = 0; j < small.size(1); ++j) {
+ printf("%.2f ", small[i][j]);
+ }
+ printf("\n");
+ }
+
+ printf("mask\n");
+ TensorContainer<cpu, 2> mask_data(Shape2(6, 8));
+ TensorContainer<cpu, 2> mask_out(Shape2(6, 8));
+ TensorContainer<cpu, 1> mask_src(Shape1(6));
+
+ mask_data = 1.0f;
+ for (int i = 0; i < 6; ++i) {
+ mask_src[i] = static_cast<float>(i);
+ }
+ mask_out = mask(mask_src, mask_data);
+ for (index_t i = 0; i < mask_out.size(0); ++i) {
+ for (index_t j = 0; j < mask_out.size(1); ++j) {
+ printf("%.2f ", mask_out[i][j]);
+ }
+ printf("\n");
+ }
+ ShutdownTensorEngine<cpu>();
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/basic_stream.cu b/3rdparty/mshadow/guide/basic_stream.cu
new file mode 100644
index 0000000..930fc54
--- /dev/null
+++ b/3rdparty/mshadow/guide/basic_stream.cu
@@ -0,0 +1,35 @@
+// header file to use mshadow
+#include "mshadow/tensor.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+int main(void) {
+ // intialize tensor engine before using tensor operation, needed for CuBLAS
+ InitTensorEngine<gpu>();
+ // create a 2 x 5 tensor, from existing space
+ Stream<gpu> *sm1 = NewStream<gpu>();
+ Stream<gpu> *sm2 = NewStream<gpu>();
+ Tensor<gpu, 2, float> ts1 = NewTensor<gpu, float>(Shape2(2, 5), 0.0f, sm1);
+ Tensor<gpu, 2, float> ts2 = NewTensor<gpu, float>(Shape2(2, 5), 0.0f, sm2);
+ ts1 = 1; // Should use stream 0.
+ ts2 = 2; // Should use stream 1. Can run in parallel with stream 0.
+ Tensor<gpu, 2> res = NewTensor<gpu, float>(Shape2(2, 2), 0.0f);
+ res.stream_ = NewStream<gpu>();
+ res = dot(ts1, ts2.T()); //Should use stream 2.
+
+ Tensor<cpu, 2> cpu_res = NewTensor<cpu, float>(Shape2(2, 2), 0.0f);
+ Copy(cpu_res, res); // default stream, should be 0.
+ for (index_t i = 0; i < cpu_res.size(0); ++i){
+ for (index_t j = 0; j < cpu_res.size(1); ++j){
+ printf("%.2f ", cpu_res[i][j]);
+ }
+ printf("\n");
+ }
+ // shutdown tensor enigne after usage
+ DeleteStream(sm1);
+ DeleteStream(sm2);
+ ShutdownTensorEngine<gpu>();
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/config.mk b/3rdparty/mshadow/guide/config.mk
new file mode 100644
index 0000000..26431bb
--- /dev/null
+++ b/3rdparty/mshadow/guide/config.mk
@@ -0,0 +1,39 @@
+#---------------------------------------------------------------------------------------
+# mshadow: the configuration compile script
+#
+# This is configuration script that you can use to compile mshadow
+# Usage:
+#
+# include config.mk in your Makefile, or directly include the definition of variables
+# include mshadow.mk after the variables are set
+#
+# Add MSHADOW_CFLAGS to the compile flags
+# Add MSHADOW_LDFLAGS to the linker flags
+# Add MSHADOW_NVCCFLAGS to the nvcc compile flags
+#----------------------------------------------------------------------------------------
+
+# whether use CUDA during compile
+USE_CUDA = 0
+
+# add the path to CUDA libary to link and compile flag
+# if you have already add them to enviroment variable, leave it as NONE
+USE_CUDA_PATH = NONE
+
+#
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas, apple
+USE_BLAS = blas
+#
+# add path to intel library, you may need it
+# for MKL, if you did not add the path to enviroment variable
+#
+USE_INTEL_PATH = NONE
+
+# whether compile with parameter server
+USE_DIST_PS = 0
+PS_PATH = NONE
+PS_THIRD_PATH = NONE
+
+# whether compile with rabit allreduce
+USE_RABIT_PS = 0
+RABIT_PATH = NONE
diff --git a/3rdparty/mshadow/guide/defop.cpp b/3rdparty/mshadow/guide/defop.cpp
new file mode 100644
index 0000000..8aa1b5c
--- /dev/null
+++ b/3rdparty/mshadow/guide/defop.cpp
@@ -0,0 +1,49 @@
+#include <cmath>
+// header file to use mshadow
+#include "mshadow/tensor.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+// user defined unary operator addone
+struct addone {
+ // map can be template function
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a) {
+ return a + static_cast<DType>(1);
+ }
+};
+// user defined binary operator max of two
+struct maxoftwo {
+ // map can also be normal functions,
+ // however, this can only be applied to float tensor
+ MSHADOW_XINLINE static float Map(float a, float b) {
+ if(a > b) return a;
+ else return b;
+ }
+};
+
+int main(void) {
+ // intialize tensor engine before using tensor operation, needed for CuBLAS
+ InitTensorEngine<cpu>();
+ // take first subscript of the tensor
+ Stream<cpu> *stream_ = NewStream<cpu>(0);
+ Tensor<cpu,2, float> mat = NewTensor<cpu>(Shape2(2,3), 0.0f, stream_);
+ Tensor<cpu,2, float> mat2= NewTensor<cpu>(Shape2(2,3), 0.0f, stream_);
+
+ mat[0][0] = -2.0f;
+ mat = F<maxoftwo>(F<addone>(mat) + 0.5f, mat2);
+
+ for (index_t i = 0; i < mat.size(0); ++i) {
+ for (index_t j = 0; j < mat.size(1); ++j) {
+ printf("%.2f ", mat[i][j]);
+ }
+ printf("\n");
+ }
+ FreeSpace(&mat); FreeSpace(&mat2);
+ DeleteStream(stream_);
+ // shutdown tensor enigne after usage
+ ShutdownTensorEngine<cpu>();
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/exp-template/.gitignore b/3rdparty/mshadow/guide/exp-template/.gitignore
new file mode 100644
index 0000000..fc070ad
--- /dev/null
+++ b/3rdparty/mshadow/guide/exp-template/.gitignore
@@ -0,0 +1 @@
+exp_*
\ No newline at end of file
diff --git a/3rdparty/mshadow/guide/exp-template/Makefile b/3rdparty/mshadow/guide/exp-template/Makefile
new file mode 100644
index 0000000..9063989
--- /dev/null
+++ b/3rdparty/mshadow/guide/exp-template/Makefile
@@ -0,0 +1,20 @@
+# set LD_LIBRARY_PATH
+export CC = gcc
+export CXX = g++
+export CFLAGS = -Wall -O3
+# specify tensor path
+BIN = exp_lazy exp_template exp_template_op
+
+.PHONY: clean all
+
+all: $(BIN)
+
+exp_lazy: exp_lazy.cpp
+exp_template: exp_template.cpp
+exp_template_op: exp_template_op.cpp
+
+$(BIN) :
+ $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^)
+
+clean:
+ rm -rf $(BIN) *~
diff --git a/3rdparty/mshadow/guide/exp-template/README.md b/3rdparty/mshadow/guide/exp-template/README.md
new file mode 100644
index 0000000..46e0019
--- /dev/null
+++ b/3rdparty/mshadow/guide/exp-template/README.md
@@ -0,0 +1,340 @@
+Expression Template Tutorial
+====
+This page explains how mshadow works. The main trick behind mshadow is called [Expression Template](http://en.wikipedia.org/wiki/Expression_templates).
+We will explain how it will affect the performance of compiled code. Expression template is the major trick behind the C++ matrix libraries such as Eigen, GSL, boost.uBLAS.
+
+How to write efficient machine learning code
+====
+Before we start, let us think of the question above. Assume we want to write down the update rule (for illustration purpose here, while typical update would be `weight += - eta * (grad + lambda * weight)` )
+```c++
+weight = - eta * (grad + lambda * weight);
+```
+Where weight and grad are vectors of length ```n```. When you choose C++ as your programming language,
+I guess the major concern is efficiency. There is one principle that is important and used in most C/C++ programs:
+* Pre-allocate necessary memory, **no temporal memory allocation** during running.
+
+An example code is like
+```c++
+void UpdateWeight (const float *grad, float eta, float lambda,
+ int n, float *weight) {
+ for (int i = 0; i < n; ++i) {
+ weight[i] = - eta * (grad[i] + lambda * weight[i]);
+ }
+}
+```
+The function takes the pre-allocated space grad, and weight, and run the calculation. Writing these functions are simple,
+however, it can be annoying when we write them repeatedly. So the question is, can we write as follows, and get same performance as previous code?
+```c++
+void UpdateWeight (const Vec& grad, float eta, float lambda, Vec& weight) {
+ weight = -eta * (grad + lambda * weight);
+}
+```
+The answer is yes, but not by the most obvious solution.
+
+A Naive Bad Solution
+====
+Let us first take a look at a most straight forward solution: operator overloading.
+```c++
+// Naive solution for vector operation overloading
+struct Vec {
+ int len;
+ float* dptr;
+ Vec(int len) : len(len) {
+ dptr = new float[len];
+ }
+ Vec(const Vec& src) : len(src.len) {
+ dptr = new float[len];
+ memcpy(dptr, src.dptr, sizeof(float)*len );
+ }
+ ~Vec(void) {
+ delete [] dptr;
+ }
+};
+
+inline Vec operator+(const Vec &lhs, const Vec &rhs) {
+ Vec res(lhs.len);
+ for (int i = 0; i < lhs.len; ++i) {
+ res.dptr[i] = lhs.dptr[i] + rhs.dptr[i];
+ }
+ return res;
+}
+```
+If we add more operators overloading in the same style, we can get what we want, and write equations instead of loop.
+However, this kind of approach is inefficient, because temporal memory is allocated and de-allocated during each operation, while we could have done better.
+
+An alternative, more effective way is only overload operator+=, operator-=, which can be implemented without temporal memory allocation. But this limits the equations we can write.
+
+We will discuss why we still need expression template although C++11 provides move assignment operator and rvalue reference at the end of this tutorial.
+
+Lazy Evaluation
+====
+Let us think why we need temporal memory allocation when doing operator+. This is because we *do not know* the target that will be assigned to in operator+,
+otherwise we could have directly storing into target memory instead of temporal memory.
+
+What if we can know the target? The following code ([exp_lazy.cpp](exp_lazy.cpp)) achieves this.
+```c++
+// Example Lazy evaluation code
+// for simplicity, we use struct and make all members public
+#include <cstdio>
+struct Vec;
+// expression structure holds the expression
+struct BinaryAddExp {
+ const Vec &lhs;
+ const Vec &rhs;
+ BinaryAddExp(const Vec &lhs, const Vec &rhs)
+ : lhs(lhs), rhs(rhs) {}
+};
+// no constructor and destructor to allocate and de-allocate memory,
+// allocation done by user
+struct Vec {
+ int len;
+ float* dptr;
+ Vec(void) {}
+ Vec(float *dptr, int len)
+ : len(len), dptr(dptr) {}
+ // here is where evaluation happens
+ inline Vec &operator=(const BinaryAddExp &src) {
+ for (int i = 0; i < len; ++i) {
+ dptr[i] = src.lhs.dptr[i] + src.rhs.dptr[i];
+ }
+ return *this;
+ }
+};
+// no evaluation happens here
+inline BinaryAddExp operator+(const Vec &lhs, const Vec &rhs) {
+ return BinaryAddExp(lhs, rhs);
+}
+
+const int n = 3;
+int main(void) {
+ float sa[n] = {1, 2, 3};
+ float sb[n] = {2, 3, 4};
+ float sc[n] = {3, 4, 5};
+ Vec A(sa, n), B(sb, n), C(sc, n);
+ // run expression
+ A = B + C;
+ for (int i = 0; i < n; ++i) {
+ printf("%d:%f==%f+%f\n", i, A.dptr[i], B.dptr[i], C.dptr[i]);
+ }
+ return 0;
+}
+```
+The idea is that we do not actually do computation in operator+, but only return a expression structure (like abstract syntax tree),
+and when we overload operator=, we see the target, as well as all the operands, and we can run computation without introducing extra memory!
+Similarly, we can define a DotExp and lazily evaluate at operator=, and redirect matrix(vector) multiplications to BLAS.
+
+
+More Lengthy Expressions and Expression Template
+====
+By using lazy evaluation, we are cool by avoiding temporal memory allocations. But the ability of the code is limited:
+* We can only write ```A=B+C```, but not more lengthy expressions.
+* When we add more expression, we need to write more operator= to evaluate each equations.
+
+Here is where the magic of template programming comes to rescue. The following code ([exp_template.cpp](exp_template.cpp)),
+which is a bit more lengthy, also allows you to write lengthy equations.
+```c++
+// Example code, expression template, and more length equations
+// for simplicity, we use struct and make all members public
+#include <cstdio>
+
+// this is expression, all expressions must inheritate it,
+// and put their type in subtype
+template<typename SubType>
+struct Exp {
+ // returns const reference of the actual type of this expression
+ inline const SubType& self(void) const {
+ return *static_cast<const SubType*>(this);
+ }
+};
+
+// binary add expression
+// note how it is inheritates from Exp
+// and put its own type into the template argument
+template<typename TLhs, typename TRhs>
+struct BinaryAddExp: public Exp<BinaryAddExp<TLhs, TRhs> > {
+ const TLhs &lhs;
+ const TRhs &rhs;
+ BinaryAddExp(const TLhs& lhs, const TRhs& rhs)
+ : lhs(lhs), rhs(rhs) {}
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return lhs.Eval(i) + rhs.Eval(i);
+ }
+};
+// no constructor and destructor to allocate
+// and de-allocate memory, allocation done by user
+struct Vec: public Exp<Vec> {
+ int len;
+ float* dptr;
+ Vec(void) {}
+ Vec(float *dptr, int len)
+ :len(len), dptr(dptr) {}
+ // here is where evaluation happens
+ template<typename EType>
+ inline Vec& operator= (const Exp<EType>& src_) {
+ const EType &src = src_.self();
+ for (int i = 0; i < len; ++i) {
+ dptr[i] = src.Eval(i);
+ }
+ return *this;
+ }
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return dptr[i];
+ }
+};
+// template add, works for any expressions
+template<typename TLhs, typename TRhs>
+inline BinaryAddExp<TLhs, TRhs>
+operator+(const Exp<TLhs> &lhs, const Exp<TRhs> &rhs) {
+ return BinaryAddExp<TLhs, TRhs>(lhs.self(), rhs.self());
+}
+
+const int n = 3;
+int main(void) {
+ float sa[n] = {1, 2, 3};
+ float sb[n] = {2, 3, 4};
+ float sc[n] = {3, 4, 5};
+ Vec A(sa, n), B(sb, n), C(sc, n);
+ // run expression, this expression is longer:)
+ A = B + C + C;
+ for (int i = 0; i < n; ++i) {
+ printf("%d:%f == %f + %f + %f\n", i,
+ A.dptr[i], B.dptr[i],
+ C.dptr[i], C.dptr[i]);
+ }
+ return 0;
+}
+```
+The key idea of the code is the template ```Exp<SubType>``` takes type of its derived class as template argument, so it can convert itself to
+the SubType via ```self()```. BinaryAddExp now is a template class that can composite expressions together, like a template version of Composite pattern.
+The evaluation is done through function Eval, which is done in a recursive way in BinaryAddExp.
+* Due to inlining, the function calls of ```src.Eval(i)``` in ```operator=``` will be compiled into ```B.dptr[i] + C.dptr[i] + C.dptr[i]``` in compile time.
+* We can write equations for element-wise operations with same efficiency as if we write a loop
+
+Make it more flexible
+====
+As we can find in the previous example, template programming is a powerful to make things flexible in compile time, our final example,
+which is closer to mshadow, allows user customized binary operators ([exp_template_op.cpp](exp_template_op.cpp)).
+```c++
+// Example code, expression template
+// with binary operator definition and extension
+// for simplicity, we use struct and make all members public
+#include <cstdio>
+
+// this is expression, all expressions must inheritate it,
+// and put their type in subtype
+template<typename SubType>
+struct Exp{
+ // returns const reference of the actual type of this expression
+ inline const SubType& self(void) const {
+ return *static_cast<const SubType*>(this);
+ }
+};
+
+// binary operators
+struct mul{
+ inline static float Map(float a, float b) {
+ return a * b;
+ }
+};
+
+// binary add expression
+// note how it is inheritates from Exp
+// and put its own type into the template argument
+template<typename OP, typename TLhs, typename TRhs>
+struct BinaryMapExp: public Exp<BinaryMapExp<OP, TLhs, TRhs> >{
+ const TLhs& lhs;
+ const TRhs& rhs;
+ BinaryMapExp(const TLhs& lhs, const TRhs& rhs)
+ :lhs(lhs), rhs(rhs) {}
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return OP::Map(lhs.Eval(i), rhs.Eval(i));
+ }
+};
+// no constructor and destructor to allocate and de-allocate memory
+// allocation done by user
+struct Vec: public Exp<Vec>{
+ int len;
+ float* dptr;
+ Vec(void) {}
+ Vec(float *dptr, int len)
+ : len(len), dptr(dptr) {}
+ // here is where evaluation happens
+ template<typename EType>
+ inline Vec& operator=(const Exp<EType>& src_) {
+ const EType &src = src_.self();
+ for (int i = 0; i < len; ++i) {
+ dptr[i] = src.Eval(i);
+ }
+ return *this;
+ }
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return dptr[i];
+ }
+};
+// template binary operation, works for any expressions
+template<typename OP, typename TLhs, typename TRhs>
+inline BinaryMapExp<OP, TLhs, TRhs>
+F(const Exp<TLhs>& lhs, const Exp<TRhs>& rhs) {
+ return BinaryMapExp<OP, TLhs, TRhs>(lhs.self(), rhs.self());
+}
+
+template<typename TLhs, typename TRhs>
+inline BinaryMapExp<mul, TLhs, TRhs>
+operator*(const Exp<TLhs>& lhs, const Exp<TRhs>& rhs) {
+ return F<mul>(lhs, rhs);
+}
+
+// user defined operation
+struct maximum{
+ inline static float Map(float a, float b) {
+ return a > b ? a : b;
+ }
+};
+
+const int n = 3;
+int main(void) {
+ float sa[n] = {1, 2, 3};
+ float sb[n] = {2, 3, 4};
+ float sc[n] = {3, 4, 5};
+ Vec A(sa, n), B(sb, n), C(sc, n);
+ // run expression, this expression is longer:)
+ A = B * F<maximum>(C, B);
+ for (int i = 0; i < n; ++i) {
+ printf("%d:%f == %f * max(%f, %f)\n",
+ i, A.dptr[i], B.dptr[i], C.dptr[i], B.dptr[i]);
+ }
+ return 0;
+}
+```
+
+Summary
+=====
+Up to this point, you should have understand basic ideas how it works:
+* Lazy evaluation, to allow us see all the operands and target
+* Template composition and recursive evaluation, to allows us evaluate arbitrary composite expressions for element-wise operations.
+* Due to template and inlining, writing expressions are as efficient as if we directly write a for loop to implement the update rule:)
+
+So write expressions when you write machine learning codes, and focus your energy on the algorithm part that matters.
+
+The Expression Template in MShadow
+=====
+Expression template in mshadow use the same key points as we introduced in the tutorial, with some minor differences:
+* We separate evaluation code from expression construction and composition code.
+ - Instead of putting Eval in Exp class. A Plan class is created from expression, and used to evaluate the result.
+ - This allows us to put less variables in Plan, for example, we do not need array length when we evaluate a data.
+ - One important reason is CUDA kernel cannot take class with const references
+ - This design choice is debatable, but we find it is useful so far.
+* Lazy support for complex expressions such as matrix dot product
+ - Besides element-wise expressions, we also want to support sugars such as ```A = dot(B.T(), C)```, again, lazy evaluation is used and no extra memory is allocated.
+* Type checking and array length checking.
+
+Notes
+====
+* Expression Template and C++11: in C++11, move constructor can be used to save repetitive allocation memory, which removes some need to expression template. However, the space still needs to be allocated at least once.
+ - This only removes the need of expression template then expression generate space, say dst = A+B+C, dst does not contain space allocated before assignment.
+ - If we want to keep the syntax that everything is pre-allocated, and expression executes without memory allocation (which is what we did in mshadow), we still need expression template.
+
diff --git a/3rdparty/mshadow/guide/exp-template/exp_lazy.cpp b/3rdparty/mshadow/guide/exp-template/exp_lazy.cpp
new file mode 100644
index 0000000..4e6a6b1
--- /dev/null
+++ b/3rdparty/mshadow/guide/exp-template/exp_lazy.cpp
@@ -0,0 +1,45 @@
+// Example Lazy evaluation code
+// for simplicity, we use struct and make all members public
+#include <cstdio>
+struct Vec;
+// expression structure holds the expression
+struct BinaryAddExp {
+ const Vec &lhs;
+ const Vec &rhs;
+ BinaryAddExp(const Vec &lhs, const Vec &rhs)
+ : lhs(lhs), rhs(rhs) {}
+};
+// no constructor and destructor to allocate and de-allocate memory,
+// allocation done by user
+struct Vec {
+ int len;
+ float* dptr;
+ Vec(void) {}
+ Vec(float *dptr, int len)
+ : len(len), dptr(dptr) {}
+ // here is where evaluation happens
+ inline Vec &operator=(const BinaryAddExp &src) {
+ for (int i = 0; i < len; ++i) {
+ dptr[i] = src.lhs.dptr[i] + src.rhs.dptr[i];
+ }
+ return *this;
+ }
+};
+// no evaluation happens here
+inline BinaryAddExp operator+(const Vec &lhs, const Vec &rhs) {
+ return BinaryAddExp(lhs, rhs);
+}
+
+const int n = 3;
+int main(void) {
+ float sa[n] = {1, 2, 3};
+ float sb[n] = {2, 3, 4};
+ float sc[n] = {3, 4, 5};
+ Vec A(sa, n), B(sb, n), C(sc, n);
+ // run expression
+ A = B + C;
+ for (int i = 0; i < n; ++i) {
+ printf("%d:%f==%f+%f\n", i, A.dptr[i], B.dptr[i], C.dptr[i]);
+ }
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/exp-template/exp_template.cpp b/3rdparty/mshadow/guide/exp-template/exp_template.cpp
new file mode 100644
index 0000000..556b103
--- /dev/null
+++ b/3rdparty/mshadow/guide/exp-template/exp_template.cpp
@@ -0,0 +1,72 @@
+// Example code, expression template, and more length equations
+// for simplicity, we use struct and make all members public
+#include <cstdio>
+
+// this is expression, all expressions must inheritate it,
+// and put their type in subtype
+template<typename SubType>
+struct Exp {
+ // returns const reference of the actual type of this expression
+ inline const SubType& self(void) const {
+ return *static_cast<const SubType*>(this);
+ }
+};
+
+// binary add expression
+// note how it is inheritates from Exp
+// and put its own type into the template argument
+template<typename TLhs, typename TRhs>
+struct BinaryAddExp: public Exp<BinaryAddExp<TLhs, TRhs> > {
+ const TLhs &lhs;
+ const TRhs &rhs;
+ BinaryAddExp(const TLhs& lhs, const TRhs& rhs)
+ : lhs(lhs), rhs(rhs) {}
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return lhs.Eval(i) + rhs.Eval(i);
+ }
+};
+// no constructor and destructor to allocate
+// and de-allocate memory, allocation done by user
+struct Vec: public Exp<Vec> {
+ int len;
+ float* dptr;
+ Vec(void) {}
+ Vec(float *dptr, int len)
+ :len(len), dptr(dptr) {}
+ // here is where evaluation happens
+ template<typename EType>
+ inline Vec& operator= (const Exp<EType>& src_) {
+ const EType &src = src_.self();
+ for (int i = 0; i < len; ++i) {
+ dptr[i] = src.Eval(i);
+ }
+ return *this;
+ }
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return dptr[i];
+ }
+};
+// template add, works for any expressions
+template<typename TLhs, typename TRhs>
+inline BinaryAddExp<TLhs, TRhs>
+operator+(const Exp<TLhs> &lhs, const Exp<TRhs> &rhs) {
+ return BinaryAddExp<TLhs, TRhs>(lhs.self(), rhs.self());
+}
+
+const int n = 3;
+int main(void) {
+ float sa[n] = {1, 2, 3};
+ float sb[n] = {2, 3, 4};
+ float sc[n] = {3, 4, 5};
+ Vec A(sa, n), B(sb, n), C(sc, n);
+ // run expression, this expression is longer:)
+ A = B + C + C;
+ for (int i = 0; i < n; ++i) {
+ printf("%d:%f == %f + %f + %f\n", i,
+ A.dptr[i], B.dptr[i],
+ C.dptr[i], C.dptr[i]);
+ }
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/exp-template/exp_template_op.cpp b/3rdparty/mshadow/guide/exp-template/exp_template_op.cpp
new file mode 100644
index 0000000..ecc1be8
--- /dev/null
+++ b/3rdparty/mshadow/guide/exp-template/exp_template_op.cpp
@@ -0,0 +1,92 @@
+// Example code, expression template
+// with binary operator definition and extension
+// for simplicity, we use struct and make all members public
+#include <cstdio>
+
+// this is expression, all expressions must inheritate it,
+// and put their type in subtype
+template<typename SubType>
+struct Exp{
+ // returns const reference of the actual type of this expression
+ inline const SubType& self(void) const {
+ return *static_cast<const SubType*>(this);
+ }
+};
+
+// binary operators
+struct mul{
+ inline static float Map(float a, float b) {
+ return a * b;
+ }
+};
+
+// binary add expression
+// note how it is inheritates from Exp
+// and put its own type into the template argument
+template<typename OP, typename TLhs, typename TRhs>
+struct BinaryMapExp: public Exp<BinaryMapExp<OP, TLhs, TRhs> >{
+ const TLhs& lhs;
+ const TRhs& rhs;
+ BinaryMapExp(const TLhs& lhs, const TRhs& rhs)
+ :lhs(lhs), rhs(rhs) {}
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return OP::Map(lhs.Eval(i), rhs.Eval(i));
+ }
+};
+// no constructor and destructor to allocate and de-allocate memory
+// allocation done by user
+struct Vec: public Exp<Vec>{
+ int len;
+ float* dptr;
+ Vec(void) {}
+ Vec(float *dptr, int len)
+ : len(len), dptr(dptr) {}
+ // here is where evaluation happens
+ template<typename EType>
+ inline Vec& operator=(const Exp<EType>& src_) {
+ const EType &src = src_.self();
+ for (int i = 0; i < len; ++i) {
+ dptr[i] = src.Eval(i);
+ }
+ return *this;
+ }
+ // evaluation function, evaluate this expression at position i
+ inline float Eval(int i) const {
+ return dptr[i];
+ }
+};
+// template binary operation, works for any expressions
+template<typename OP, typename TLhs, typename TRhs>
+inline BinaryMapExp<OP, TLhs, TRhs>
+F(const Exp<TLhs>& lhs, const Exp<TRhs>& rhs) {
+ return BinaryMapExp<OP, TLhs, TRhs>(lhs.self(), rhs.self());
+}
+
+template<typename TLhs, typename TRhs>
+inline BinaryMapExp<mul, TLhs, TRhs>
+operator*(const Exp<TLhs>& lhs, const Exp<TRhs>& rhs) {
+ return F<mul>(lhs, rhs);
+}
+
+// user defined operation
+struct maximum{
+ inline static float Map(float a, float b) {
+ return a > b ? a : b;
+ }
+};
+
+const int n = 3;
+int main(void) {
+ float sa[n] = {1, 2, 3};
+ float sb[n] = {2, 3, 4};
+ float sc[n] = {3, 4, 5};
+ Vec A(sa, n), B(sb, n), C(sc, n);
+ // run expression, this expression is longer:)
+ A = B * F<maximum>(C, B);
+ for (int i = 0; i < n; ++i) {
+ printf("%d:%f == %f * max(%f, %f)\n",
+ i, A.dptr[i], B.dptr[i], C.dptr[i], B.dptr[i]);
+ }
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/mshadow-ps/.gitignore b/3rdparty/mshadow/guide/mshadow-ps/.gitignore
new file mode 100644
index 0000000..351b5b4
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/.gitignore
@@ -0,0 +1,4 @@
+log
+*cpu
+*gpu
+core*
diff --git a/3rdparty/mshadow/guide/mshadow-ps/2-levels.png b/3rdparty/mshadow/guide/mshadow-ps/2-levels.png
new file mode 100644
index 0000000..4f8a0e8
Binary files /dev/null and b/3rdparty/mshadow/guide/mshadow-ps/2-levels.png differ
diff --git a/3rdparty/mshadow/guide/mshadow-ps/Makefile b/3rdparty/mshadow/guide/mshadow-ps/Makefile
new file mode 100644
index 0000000..58d64a3
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/Makefile
@@ -0,0 +1,45 @@
+# set LD_LIBRARY_PATH
+export CC = gcc
+export CXX = g++
+export NVCC =nvcc
+include config.mk
+include ../../make/mshadow.mk
+export CFLAGS = -Wall -O3 -std=c++11 -fopenmp -I../../ $(MSHADOW_CFLAGS)
+export LDFLAGS= -lm $(MSHADOW_LDFLAGS)
+export NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
+
+# specify tensor path local_sum.cpu
+BIN = local_sum.cpu
+OBJ =
+CUOBJ =
+CUBIN = local_sum.gpu
+
+ifeq ($(USE_DIST_PS),1)
+BIN = dist_async_sum.cpu
+LDFLAGS += -lunwind
+endif
+
+.PHONY: clean all
+
+all: $(BIN) #$(CUBIN)
+
+local_sum.cpu: local_sum.cpp
+local_sum.gpu: local_sum.cu
+
+dist_async_sum.cpu: dist_async_sum.cpp dist_async_sum-inl.h
+dist_sync_sum.cpu: dist_sync_sum.cpp
+
+$(BIN) :
+ $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^) $(LDFLAGS) $(PS_LIB)
+
+$(OBJ) :
+ $(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
+
+$(CUOBJ) :
+ $(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" $(filter %.cu, $^)
+
+$(CUBIN) :
+ $(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" -Xlinker "$(LDFLAGS)" $(filter %.cu %.cpp %.o, $^)
+
+clean:
+ $(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~
diff --git a/3rdparty/mshadow/guide/mshadow-ps/README.md b/3rdparty/mshadow/guide/mshadow-ps/README.md
new file mode 100644
index 0000000..146c7f0
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/README.md
@@ -0,0 +1,227 @@
+mshadow-ps
+====
+### Parameter Server Interface for GPU Tensor
+
+mshadow-ps provides asynchronize parameter server interface for mshadow GPU/CPU Tensor.
+This allows you to do ***multi-GPU*** and ***disrtibuted*** (deep) learning in
+an ***easy*** and ***unified*** way.
+
+mshadow-ps implemented a two-level parameter server. The architecture is shown
+in the following figure. Typically, a GPU card or a cpu core runs a worker node,
+then a level-1 server node communicates with the worker nodes on the same
+machine. Inter-machine communication is then via the level-2 server nodes.
+
+The rational is that both the bandwidth and latency between the worker nodes in
+a single machine is usually 10x better than the inter-machine ones. By using the
+two level parameter server, we can use different consistency models at different
+level to better trade-off the algorithm efficiency and system performance. For
+example, we can use a sequential consistency model, also known as
+[BSP](http://en.wikipedia.org/wiki/Bulk_synchronous_parallel), on level 1 for
+guaranteed algorithm convergence, but use a
+[eventual consistency model](http://en.wikipedia.org/wiki/Eventual_consistency)
+on level 2 to hide the network latency. See our
+[OSDI'14 paper](https://www.usenix.org/conference/osdi14/technical-sessions/presentation/li_mu)
+for more details.
+
+
+
+####List of Resources
+* [API Documentation](http://homes.cs.washington.edu/~tqchen/mshadow/doc/namespacemshadow_1_1mshadow_ps.html)
+* [Library Interface Header](../../mshadow-ps/mshadow_ps.h)
+* Tutorial in this page
+
+Working with Level-1 Server
+====
+Suppose that we are now implementing a Multi-GPU learning program.
+One way to do that is through data parallelism. We can launch many
+threads, with each thread compute gradient on one GPU, and aggregate
+the statistics together.
+However, the gradient synchronization step could be cost time, and in
+many cases, we can do the computation in an smarter way, so that
+we ***overlaps the computation with the synchronization***.
+
+mshadow-ps provides interface to do such synchronization in an easy way.
+The following documents provides a way
+
+### Getting Sum from Multiple GPUs
+We first get familiar with the interface of mshadow-mshadow_ps. Through the following
+program in [local_sum-inl.h](local_sum-inl.h). You can compile the program
+by setup the [config.mk](config.mk) according to your computers's enviroment, and type make.
+
+In the following program, each thread first does some computation locally, then tries to get the sum
+of ```data``` through mshadow-ps interface.
+There are four key functions in ```ISharedModel``` interface
+* [InitKey](../../mshadow-ps/mshadow_ps.h#L76) allocates a key to specific tensor shape
+* [Push](../../mshadow-ps/mshadow_ps.h#L100) pushes out the local data to the synchronization interface
+ - The data pushed by different devices will be aggregated together by key
+ - Push is an asynchronize call and returns immediately
+* [PullReq](../../mshadow-ps/mshadow_ps.h#L122) requests the result of synchronization to be copied back
+ - In the local default case, the synchronized result is the sum of pushed data
+ - mshadow-ps also support the weight update on server side, where the result of PullReq is the updated weight instead of sum of gradient
+ - PullReq is also asynchronize
+* [PullWait](../../mshadow-ps/mshadow_ps.h#L87) wait until the pull request of corresponding key finishes
+
+```c++
+// this function is runed by specific thread
+template<typename xpu>
+inline void RunWorkerThread(int devid,
+ mshadow::ps::ISharedModel<xpu, float> *ps) {
+ // initialize tensor engine
+ mshadow::InitTensorEngine<xpu>(devid);
+ mshadow::Stream<xpu> *stream = mshadow::NewStream<xpu>();
+ // allocate tensor on xpu
+ mshadow::TensorContainer<xpu, 2> data(mshadow::Shape2(2, 3));
+ // set the computation stream to the new allocated stream
+ // this will make subsequent computation whose target is data
+ // to use the stream, stream is needed for async execution in GPU
+ data.set_stream(stream);
+ // assume these operations sets the content of dataient
+ data[0] = 1.0f;
+ data[1] = devid + data[0];
+ printf("dev%d: before sync, data:\n", devid);
+ // use print to show result, do not call
+ // print normally since Copy will block
+ Print(data);
+ printf("====================\n");
+ // intiaialize the key, register the shape on parameter server
+ ps->InitKey(data[0].shape_, 0, devid);
+ ps->InitKey(data[1].shape_, 1, devid);
+ // push data[0] out, for update, or aggregation
+ // 0 is the key of the data, devid is the current device id
+ ps->Push(data[0], 0, devid);
+ // pull request is used to request the data to be copied back
+ // once computation is done
+ ps->PullReq(data[0], 0, devid);
+ // computation can be done here..
+ // the pull request handler will be overlapped with
+ // similar as previous call
+ ps->Push(data[1], 1, devid);
+ ps->PullReq(data[1], 1, devid);
+ // more computation can be done here...
+ // the computation will be overlapped
+ // PullWait will block until these request finishes
+ ps->PullWait(0, devid);
+ ps->PullWait(1, devid);
+ printf("dev%d: after sync, data:\n", devid);
+ // use print to show result, do not call
+ // print normally since Copy will block
+ Print(data);
+ printf("====================\n");
+ mshadow::DeleteStream(stream);
+ mshadow::ShutdownTensorEngine<xpu>();
+}
+
+template<typename xpu>
+inline int Run(int argc, char *argv[]) {
+ if (argc < 2) {
+ printf("Usage: device list\n"\
+ "\tfor CPU the device list can be arbitrary\n"\
+ "\tfor GPU the device list need to be actual device index\n");
+ return 0;
+ }
+ // list of device ids
+ std::vector<int> devs;
+ // initialization
+ for (int i = 1; i < argc; ++i) {
+ // record the device id
+ devs.push_back(atoi(argv[i]));
+ }
+ mshadow::ps::ISharedModel<xpu, float>
+ *ps = mshadow::ps::CreateSharedModel<xpu, float>("local");
+ // intiaialize the ps
+ ps->Init(devs);
+ // use openmp to launch #devs threads
+ #pragma omp parallel num_threads(devs.size())
+ {
+ int tid = omp_get_thread_num();
+ RunWorkerThread<xpu>(devs[tid], ps);
+ }
+ delete ps;
+ return 0;
+}
+```
+In the above example, we did not do weight update on server side, so the synchronization result is
+simply the sum of data on each device. The key property of this interface is that the Push and PullReq are asynchronize.
+* We can call these two functions once the gradient is ready, and the mshadow-ps will do the data synchronization in the background.
+* When we need the result of synchronization, we simply call PullWait to wait the synchronization task to finish.
+* Such interface allows us to do additional computation between the Push/PullReq and PullWait
+
+### A MultiGPU Neural Net
+To get a more concrete understanding of the interface. We give an example of multi-GPU two layer neuralnet
+in [../neuralnet/nnet_ps.cu](../neuralnet/nnet_ps.cu). The general idea is follows
+* Push and PullReq is called once we get the gradient of certain layer
+* PullWait is called before we do forward on that layer next time
+* This creates a ***time lag*** between the backprop and next forward to that layer
+ - mshadow-ps do synchronization concurrently with computations during the time lag
+ - The time lag is big for latter layers, which also usually need more time to synchronize
+
+There are several note of the mshadow-ps on the neural net code
+* Callback function in PullReq
+ - A callback function can be pass to PullReq to be called when the request complete
+ - We place weight update in the callback to perform update when we get the gradient sum
+* Computing stream
+ - Due to GPU's programming model, we need to do computation on non-default stream
+ - Use set_stream in mshadow tensors to set stream to computation stream
+ - To report error when you did not use stream, you can compile with -DMSHADOW_FORCE_STREAM
+
+We should note thate because the example runs on MNIST, which is an quite small dataset, you may not observe
+speedup with multiple cards. However, you will find significant speedup when you run on other tasks.
+The newest version of [cxxnet](https://github.com/antinucleon/cxxnet)
+
+### Moving Parameter Update to the Server
+In all the examples so far, we use mshadow-ps to get the aggregated sum of gradients, and update
+weights locally on each GPU. For more advanced usage of mshadow-ps, we can move the weight update
+to the server. The communication pattern is as follows
+* Each thread still call Push to push out gradient
+* The server will apply the update rule to update the weight
+* Each thread call PullReq to pull back the weight from server
+
+Such update pattern is suitable under distributed setting. To do so, user need to implement an
+[IModelUpdater](../../mshadow-ps/mshadow_ps.h#L202) interface. And define the following CreateModelUpdater function
+in the program
+```c++
+namespace mshadow {
+namespace ps {
+template<>
+IModelUpdater<float> *CreateModelUpdater() {
+ return new MyModelUpdater();
+}
+}
+}
+```
+Before calling ISharedModel.Init, user need to call ```ps->SetParam("update_on_server", "1")``` to set the update
+mode on the server side. If user uses distributed shared model, user must define ModelUpdater.
+
+Working with Level-2 Server
+====
+
+First build the parameter server (replace `ps_dir` to any convenient directory)
+
+```bash
+git clone https://github.com/dmlc/parameter_server -b dev ps_dir
+cd ps_dir
+./script/install_third.sh
+make -j8
+```
+
+Next change `config.mk` to
+```bash
+USE_DIST_PS = 1
+PS_PATH = ps_dir
+```
+
+Then `make`.
+
+Next start 1 server node, 3 worker nodes with 2 devices in each worker node:
+```bash
+./local.sh 1 3 ./dist_async_sum.cpu 1 2
+```
+
+The `dist_async_sum-inl.h` is similar to `local_sum-inl.h`. The main differences
+are 1) we create the server at a remote node, and set
+`update_on_server` to be true.
+```c++
+auto* ps = mshadow::ps::CreateSharedModel<xpu, float>("dist");
+ps->SetParam("update_on_server", "1");
+```
+2) we explicitly create server node and worker node at `dist_async_sum.cpp`
diff --git a/3rdparty/mshadow/guide/mshadow-ps/config.mk b/3rdparty/mshadow/guide/mshadow-ps/config.mk
new file mode 100644
index 0000000..ed7af28
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/config.mk
@@ -0,0 +1,40 @@
+#---------------------------------------------------------------------------------------
+# mshadow: the configuration compile script
+#
+# This is configuration script that you can use to compile mshadow
+# Usage:
+#
+# include config.mk in your Makefile, or directly include the definition of variables
+# include mshadow.mk after the variables are set
+#
+# Add MSHADOW_CFLAGS to the compile flags
+# Add MSHADOW_LDFLAGS to the linker flags
+# Add MSHADOW_NVCCFLAGS to the nvcc compile flags
+#----------------------------------------------------------------------------------------
+
+# whether use CUDA during compile
+USE_CUDA = 0
+
+# add the path to CUDA libary to link and compile flag
+# if you have already add them to enviroment variable, leave it as NONE
+USE_CUDA_PATH = NONE
+
+#
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas, apple
+USE_BLAS = blas
+#
+# add path to intel library, you may need it
+# for MKL, if you did not add the path to enviroment variable
+#
+USE_INTEL_PATH = NONE
+
+# whether compile with parameter server
+USE_DIST_PS = 0
+PS_PATH = ../../../parameter_server/
+PS_THIRD_PATH = NONE
+
+# whether compile with rabit allreduce
+USE_RABIT_PS = 1
+RABIT_PATH = ../../rabit
+
diff --git a/3rdparty/mshadow/guide/mshadow-ps/dbstr.h b/3rdparty/mshadow/guide/mshadow-ps/dbstr.h
new file mode 100644
index 0000000..59a7135
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/dbstr.h
@@ -0,0 +1,35 @@
+#pragma once
+#include <mshadow/tensor.h>
+#include <sstream>
+
+template<typename DType>
+std::string dbstr(mshadow::Tensor<mshadow::cpu, 1, DType> ts) {
+ std::stringstream ss;
+ for (mshadow::index_t i = 0; i < ts.size(0); ++i)
+ ss << ts[i] << " ";
+ ss << "\n";
+ return ss.str();
+}
+
+template<typename DType>
+std::string dbstr(mshadow::Tensor<mshadow::cpu, 2, DType> ts) {
+ std::stringstream ss;
+ for (mshadow::index_t i = 0; i < ts.size(0); ++i) {
+ for (mshadow::index_t j = 0; j < ts.size(1); ++j) {
+ ss << ts[i][j] << " ";
+ }
+ ss << "\n";
+ }
+ ss << "\n";
+ return ss.str();
+}
+
+template<typename DType>
+std::string dbstr(mshadow::Tensor<mshadow::cpu, 3, DType> ts) {
+ std::stringstream ss;
+ for (mshadow::index_t i = 0; i < ts.size(0); ++i) {
+ ss << dbstr(ts[i]) << "\n";
+ }
+ ss << "\n";
+ return ss.str();
+}
diff --git a/3rdparty/mshadow/guide/mshadow-ps/dist_async_sum-inl.h b/3rdparty/mshadow/guide/mshadow-ps/dist_async_sum-inl.h
new file mode 100644
index 0000000..9b60460
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/dist_async_sum-inl.h
@@ -0,0 +1,124 @@
+/**
+ * @brief Simple test of KVLayer
+ */
+#include "ps.h"
+#include "parameter/kv_layer.h"
+#include <cstdio>
+#include <iostream>
+#include <omp.h>
+#include <map>
+#include <mshadow/tensor.h>
+#include <mshadow-ps/mshadow_ps.h>
+#include "dbstr.h"
+#include "glog/logging.h"
+
+namespace mshadow {
+namespace ps {
+
+
+template<typename DType>
+class Updater : public IModelUpdater<DType> {
+ protected:
+ void InitModel_(int key, Tensor<cpu, 1, DType> data) {
+ data = 0;
+ data_[key] = data;
+ }
+
+ void Update_(int key, Tensor<cpu, 1, DType> data) {
+ data_[key] += data;
+ // LOG(ERROR) << dbstr(data_[key]);
+ }
+ std::map<int, Tensor<cpu, 1, DType> > data_;
+};
+
+template<typename DType>
+IModelUpdater<DType> *CreateModelUpdater(void) {
+ return new Updater<DType>();
+}
+
+} // namespace ps
+} // namespace mshadow
+
+// this function is runed by specific thread
+template<typename xpu>
+inline void RunWorkerThread(int devid,
+ mshadow::ps::ISharedModel<xpu, float> *ps) {
+ // initialize tensor engine
+ mshadow::InitTensorEngine<xpu>(devid);
+ mshadow::Stream<xpu> *stream = mshadow::NewStream<xpu>();
+ // allocate tensor on xpu
+ mshadow::TensorContainer<xpu, 2> data(mshadow::Shape2(2, 3));
+ // set the computation stream to the new allocated stream
+ // this will make subsequent computation whose target is data
+ // to use the stream, stream is needed for async execution in GPU
+ data.set_stream(stream);
+ // intiaialize the key, register the shape on parameter server
+ ps->InitKey(data[0].shape_, 0, devid);
+ ps->InitKey(data[1].shape_, 1, devid);
+ // first step, pull the data back from server
+ ps->PullReq(data[0], 0, devid);
+ ps->PullReq(data[1], 1, devid);
+
+ // PullWait will block until these request finishes
+ ps->PullWait(0, devid);
+ ps->PullWait(1, devid);
+
+ data[1] = devid + data[0];
+
+ LOG(ERROR) << "node " << ::ps::MyNodeID() << ", dev " << devid << ": before sync\n"
+ << dbstr(data);
+
+ // push data[0] out, for update, or aggregation
+ // 0 is the key of the data, devid is the current device id
+ ps->Push(data[0], 0, devid);
+ // pull request is used to request the data to be copied back
+ // once computation is done
+ ps->PullReq(data[0], 0, devid);
+ // computation can be done here..
+ // the pull request handler will be overlapped with
+ // similar as previous call
+ ps->PullWait(0, devid);
+
+ ps->Push(data[1], 1, devid);
+ ps->PullReq(data[1], 1, devid);
+ // more computation can be done here...
+ // the computation will be overlapped
+ // PullWait will block until these request finishes
+ ps->PullWait(1, devid);
+
+ LOG(ERROR) << "node " << ::ps::MyNodeID() << ", dev " << devid
+ << ": after sync\n" << dbstr(data);
+
+ mshadow::DeleteStream(stream);
+ mshadow::ShutdownTensorEngine<xpu>();
+}
+
+template<typename xpu>
+inline int Run(int argc, char *argv[]) {
+ if (argc < 2) {
+ printf("Usage: device list\n"\
+ "\tfor CPU the device list can be arbitrary\n"\
+ "\tfor GPU the device list need to be actual device index\n");
+ return 0;
+ }
+ // list of device ids
+ std::vector<int> devs;
+ // initialization
+ for (int i = 1; i < argc; ++i) {
+ // record the device id
+ devs.push_back(atoi(argv[i]));
+ }
+ mshadow::ps::ISharedModel<xpu, float>
+ *ps = mshadow::ps::CreateSharedModel<xpu, float>("dist");
+ // intiaialize the ps
+ ps->SetParam("update_on_server", "1");
+ ps->Init(devs);
+ // use openmp to launch #devs threads
+ #pragma omp parallel num_threads(devs.size())
+ {
+ int tid = omp_get_thread_num();
+ RunWorkerThread<xpu>(devs[tid], ps);
+ }
+ delete ps;
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/mshadow-ps/dist_async_sum.cpp b/3rdparty/mshadow/guide/mshadow-ps/dist_async_sum.cpp
new file mode 100644
index 0000000..1189105
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/dist_async_sum.cpp
@@ -0,0 +1,11 @@
+#include "./dist_async_sum-inl.h"
+
+int CreateServerNode(int argc, char *argv[]) {
+ mshadow::ps::MShadowServerNode<float> server(argc, argv);
+ return 0;
+}
+
+
+int WorkerNodeMain(int argc, char *argv[]) {
+ return Run<mshadow::cpu>(argc, argv);
+}
diff --git a/3rdparty/mshadow/guide/mshadow-ps/local.sh b/3rdparty/mshadow/guide/mshadow-ps/local.sh
new file mode 100755
index 0000000..a965ea8
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/local.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# set -x
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../third_party/lib
+if [ $# -lt 3 ]; then
+ echo "usage: $0 num_servers num_workers bin [args..]"
+ exit -1;
+fi
+
+num_servers=$1
+shift
+num_workers=$1
+shift
+bin=$1
+shift
+arg="-num_servers ${num_servers} -num_workers ${num_workers} -log_dir log $@"
+
+
+# killall -q $(basename ${bin})
+# killall -q ${bin}
+
+# start the scheduler
+Sch="role:SCHEDULER,hostname:'127.0.0.1',port:8001,id:'H'"
+${bin} -my_node ${Sch} -scheduler ${Sch} ${arg} &
+
+# start servers
+for ((i=0; i<${num_servers}; ++i)); do
+ port=$((9600 + ${i}))
+ N="role:SERVER,hostname:'127.0.0.1',port:${port},id:'S${i}'"
+ ${bin} -my_node ${N} -scheduler ${Sch} ${arg} &
+done
+
+# start workers
+for ((i=0; i<${num_workers}; ++i)); do
+ port=$((9500 + ${i}))
+ N="role:WORKER,hostname:'127.0.0.1',port:${port},id:'W${i}'"
+ ${bin} -my_node ${N} -scheduler ${Sch} ${arg} &
+done
+
+wait
diff --git a/3rdparty/mshadow/guide/mshadow-ps/local_sum-inl.h b/3rdparty/mshadow/guide/mshadow-ps/local_sum-inl.h
new file mode 100644
index 0000000..cfd5037
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/local_sum-inl.h
@@ -0,0 +1,119 @@
+// This is an example demonstrating the usage of mshadow ps
+#include <cstdio>
+// use openmp to launch multiple threads
+#include <omp.h>
+#include <mshadow/tensor.h>
+#include <mshadow-ps/mshadow_ps.h>
+
+// simple util to print result
+void Print_(mshadow::Tensor<mshadow::cpu, 2, float> ts) {
+ for (mshadow::index_t i = 0; i < ts.size(0); ++i) {
+ for (mshadow::index_t j = 0; j < ts.size(1); ++j) {
+ printf("%g ", ts[i][j]);
+ }
+ printf("\n");
+ }
+}
+template<typename xpu>
+inline void Print(mshadow::Tensor<xpu, 2, float> ts) {
+ mshadow::TensorContainer<mshadow::cpu, 2, float> tmp;
+ tmp.Resize(ts.shape_);
+ mshadow::Copy(tmp, ts);
+ Print_(tmp);
+}
+
+// this function is runed by specific thread
+template<typename xpu>
+inline void RunWorkerThread(int devid,
+ mshadow::ps::ISharedModel<xpu, float> *ps) {
+ // initialize tensor engine
+ mshadow::InitTensorEngine<xpu>(devid);
+ mshadow::Stream<xpu> *stream = mshadow::NewStream<xpu>(devid);
+ // allocate tensor on xpu
+ mshadow::TensorContainer<xpu, 2> data(mshadow::Shape2(2, 3));
+ // set the computation stream to the new allocated stream
+ // this will make subsequent computation whose target is data
+ // to use the stream, stream is needed for async execution in GPU
+ data.set_stream(stream);
+ // assume these operations sets the content of dataient
+ data[0] = 1.0f;
+ data[1] = devid + data[0];
+ printf("dev%d: before sync, data:\n", devid);
+ // use print to show result, do not call
+ // print normally since Copy will block
+ Print(data);
+ printf("====================\n");
+ // intiaialize the key, register the shape on parameter server
+ ps->InitKey(data[0].shape_, 0, devid);
+ ps->InitKey(data[1].shape_, 1, devid);
+ // push data[0] out, for update, or aggregation
+ // 0 is the key of the data, devid is the current device id
+ ps->Push(data[0], 0, devid);
+ // pull request is used to request the data to be copied back
+ // once computation is done
+ ps->PullReq(data[0], 0, devid);
+ // computation can be done here..
+ // the pull request handler will be overlapped with
+ // similar as previous call
+ ps->Push(data[1], 1, devid);
+ ps->PullReq(data[1], 1, devid);
+ // more computation can be done here...
+ // the computation will be overlapped
+ // PullWait will block until these request finishes
+ ps->PullWait(0, devid);
+ ps->PullWait(1, devid);
+ printf("dev%d: after sync, data:\n", devid);
+ // use print to show result, do not call
+ // print normally since Copy will block
+ Print(data);
+ printf("====================\n");
+ mshadow::DeleteStream(stream);
+ mshadow::ShutdownTensorEngine<xpu>();
+}
+
+namespace mshadow {
+namespace ps {
+// model updater is used when update is happening on server side
+// if we only use parameter server for sum aggregation
+// this is not needed, but we must declare this function to return NULL
+template<>
+IModelUpdater<float> *CreateModelUpdater(void) {
+ return NULL;
+}
+}
+}
+
+template<typename xpu>
+inline int Run(int argc, char *argv[]) {
+ if (argc < 2) {
+ printf("Usage: device list\n"\
+ "\tfor CPU the device list can be arbitrary\n"\
+ "\tfor GPU the device list need to be actual device index\n");
+ return 0;
+ }
+#if MSHADOW_RABIT_PS
+ rabit::Init(argc, argv);
+#endif
+ // list of device ids
+ std::vector<int> devs;
+ // initialization
+ for (int i = 1; i < argc; ++i) {
+ // record the device id
+ devs.push_back(atoi(argv[i]));
+ }
+ mshadow::ps::ISharedModel<xpu, float>
+ *ps = mshadow::ps::CreateSharedModel<xpu, float>("local");
+ // intiaialize the ps
+ ps->Init(devs);
+ // use openmp to launch #devs threads
+ #pragma omp parallel num_threads(devs.size())
+ {
+ int tid = omp_get_thread_num();
+ RunWorkerThread<xpu>(devs[tid], ps);
+ }
+ delete ps;
+#if MSHADOW_RABIT_PS
+ rabit::Finalize();
+#endif
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/mshadow-ps/local_sum.cpp b/3rdparty/mshadow/guide/mshadow-ps/local_sum.cpp
new file mode 100644
index 0000000..7f0eed0
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/local_sum.cpp
@@ -0,0 +1,4 @@
+#include "./local_sum-inl.h"
+int main(int argc, char *argv[]) {
+ return Run<mshadow::cpu>(argc, argv);
+}
diff --git a/3rdparty/mshadow/guide/mshadow-ps/local_sum.cu b/3rdparty/mshadow/guide/mshadow-ps/local_sum.cu
new file mode 100644
index 0000000..6e83960
--- /dev/null
+++ b/3rdparty/mshadow/guide/mshadow-ps/local_sum.cu
@@ -0,0 +1,4 @@
+#include "./local_sum-inl.h"
+int main(int argc, char *argv[]) {
+ return Run<mshadow::gpu>(argc, argv);
+}
diff --git a/3rdparty/mshadow/guide/neuralnet/Makefile b/3rdparty/mshadow/guide/neuralnet/Makefile
new file mode 100644
index 0000000..826384b
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/Makefile
@@ -0,0 +1,38 @@
+# set LD_LIBRARY_PATH
+export CC = gcc
+export CXX = g++
+export NVCC =nvcc
+include config.mk
+include ../../make/mshadow.mk
+export CFLAGS = -Wall -O3 -I../../ -fopenmp $(MSHADOW_CFLAGS)
+export LDFLAGS= -lm $(MSHADOW_LDFLAGS)
+export NVCCFLAGS = -O3 --use_fast_math -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
+
+# specify tensor path
+BIN =
+OBJ =
+CUOBJ =
+CUBIN = nnet convnet nnet_ps
+.PHONY: clean all
+
+all: $(BIN) $(OBJ) $(CUBIN) $(CUOBJ)
+
+nnet: nnet.cu
+nnet_ps: nnet_ps.cu
+convnet: convnet.cu
+
+$(BIN) :
+ $(CXX) $(CFLAGS) -o $@ $(filter %.cpp %.o %.c, $^) $(LDFLAGS)
+
+$(OBJ) :
+ $(CXX) -c $(CFLAGS) -o $@ $(firstword $(filter %.cpp %.c, $^) )
+
+$(CUOBJ) :
+ $(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" $(filter %.cu, $^)
+
+$(CUBIN) :
+ $(NVCC) -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" -Xlinker "$(LDFLAGS)" $(filter %.cu %.cpp %.o, $^)
+
+clean:
+ $(RM) $(OBJ) $(BIN) $(CUBIN) $(CUOBJ) *~
+
diff --git a/3rdparty/mshadow/guide/neuralnet/README.md b/3rdparty/mshadow/guide/neuralnet/README.md
new file mode 100644
index 0000000..dd181e7
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/README.md
@@ -0,0 +1,16 @@
+Example Neural Net code with MShadow
+====
+
+To compile the code, modify ```config.mk``` to the setting you like and type make
+* You will need to have CUDA and a version of BLAS
+
+To run the demo, download MNIST dataset from: http://yann.lecun.com/exdb/mnist/
+unzip all the files into current folder
+
+and run by ./nnet cpu or ./nnet gpu. ./convnet cpu or ./convnet gpu
+
+MultiGPU Version
+====
+* If you have two GPUs, you can run it by ```./nnet_ps gpu 0 1```.
+* You can run it using CPUs ```./nnet_ps cpu 0 1```.
+* This is an demonstration of mshadow-ps interface, see introduction in [../mshadow-ps](../mshadow-ps)
diff --git a/3rdparty/mshadow/guide/neuralnet/config.mk b/3rdparty/mshadow/guide/neuralnet/config.mk
new file mode 100644
index 0000000..6c10b79
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/config.mk
@@ -0,0 +1,35 @@
+#---------------------------------------------------------------------------------------
+# mshadow: the configuration compile script
+#
+# This is configuration script that you can use to compile mshadow
+# Usage:
+#
+# include config.mk in your Makefile, or directly include the definition of variables
+# include mshadow.mk after the variables are set
+#
+# Add MSHADOW_CFLAGS to the compile flags
+# Add MSHADOW_LDFLAGS to the linker flags
+# Add MSHADOW_NVCCFLAGS to the nvcc compile flags
+#----------------------------------------------------------------------------------------
+
+# whether use CUDA during compile
+USE_CUDA = 1
+
+# add the path to CUDA libary to link and compile flag
+# if you have already add them to enviroment variable, leave it as NONE
+USE_CUDA_PATH = NONE
+
+#
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas, apple
+USE_BLAS = openblas
+#
+# add path to intel library, you may need it
+# for MKL, if you did not add the path to enviroment variable
+#
+USE_INTEL_PATH = NONE
+
+# whether compile with parameter server
+USE_DIST_PS = 0
+PS_PATH = NONE
+PS_THIRD_PATH = NONE
diff --git a/3rdparty/mshadow/guide/neuralnet/convnet.cu b/3rdparty/mshadow/guide/neuralnet/convnet.cu
new file mode 100644
index 0000000..5276338
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/convnet.cu
@@ -0,0 +1,282 @@
+// this implements a simple convolution neural net: conv-maxpool-fullc
+#include <vector>
+// header file to use mshadow
+#include "mshadow/tensor.h"
+// helper function to load mnist dataset
+#include "util.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+// define operations
+struct relu{
+ MSHADOW_XINLINE static real_t Map(real_t a) {
+ using namespace std;
+ return max(a, 0.0f);
+ }
+};
+struct relu_grad {
+ MSHADOW_XINLINE static real_t Map(real_t a) {
+ return a > 0.0f ? 1.0f : 0.0f;
+ }
+};
+
+/*! \brief interface for nnet, interfacd allows use to use GPU/CPU implementation in a unified way */
+class INNet{
+ public:
+ virtual void Forward(const Tensor<cpu, 4, real_t>& inbatch, Tensor<cpu, 2, real_t> &oubatch) = 0;
+ virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) = 0;
+ virtual void Update(void) = 0;
+ virtual ~INNet() {}
+};
+
+/*!
+ * \brief simple two layer conv-net conv-pool-flat-fullc
+ * this implementation is device invariant
+ */
+template<typename xpu>
+class ConvNet : public INNet {
+ public:
+ // initialize the network
+ ConvNet(int batch_size, int insize, int nchannel, int ksize, int kstride, int psize, int num_out)
+ :rnd(0), ksize(ksize), kstride(kstride), psize(psize) {
+ // setup stream
+ Stream<xpu> *stream = NewStream<xpu>();
+ ninput.set_stream(stream);
+ nhidden.set_stream(stream);
+ nhiddenbak.set_stream(stream);
+ npool.set_stream(stream);
+ npoolbak.set_stream(stream);
+ nflat.set_stream(stream);
+ nout.set_stream(stream);
+ hbias.set_stream(stream); g_hbias.set_stream(stream);
+ obias.set_stream(stream); g_obias.set_stream(stream);
+ Ki2h.set_stream(stream); g_Ki2h.set_stream(stream);
+ Wh2o.set_stream(stream); g_Wh2o.set_stream(stream);
+ tmp_col.set_stream(stream);
+ tmp_dst.set_stream(stream);
+ // setup nodes
+ ninput.Resize(Shape4(batch_size, 1, insize, insize));
+ nhidden.Resize(Shape4(batch_size, nchannel, (insize - ksize)/kstride+1, (insize -ksize)/kstride+1));
+ nhiddenbak.Resize(nhidden.shape_);
+ npool.Resize(Shape4(batch_size, nchannel, (nhidden.size(2)+1-psize)/psize, (nhidden.size(3)+1-psize)/psize));
+ npoolbak.Resize(npool.shape_);
+ nflat.Resize(Shape2(batch_size, npool.size(1)*npool.size(2)*npool.size(3)));
+ nout.Resize(Shape2(batch_size, num_out));
+ // setup bias
+ hbias.Resize(Shape1(nchannel)); g_hbias.Resize(hbias.shape_);
+ obias.Resize(Shape1(num_out)); g_obias.Resize(obias.shape_);
+ hbias = 0.0f; obias = 0.0f;
+ // setup weights
+ Ki2h.Resize(Shape2(nchannel, ksize*ksize)); g_Ki2h.Resize(Ki2h.shape_);
+ Wh2o.Resize(Shape2(nflat.size(1), num_out)); g_Wh2o.Resize(Wh2o.shape_);
+ rnd.SampleGaussian(&Ki2h, 0, 0.01f);
+ rnd.SampleGaussian(&Wh2o, 0, 0.01f);
+
+ printf("conv=%d, pool=%d\n", nhidden.size(3), npool.size(3));
+ }
+ virtual ~ConvNet() {}
+ // forward propagation
+ virtual void Forward(const Tensor<cpu, 4, real_t>& inbatch, Tensor<cpu, 2, real_t> &oubatch) {
+ index_t batch_size = inbatch.size(0);
+ // copy data to input layer
+ Copy(ninput, inbatch, ninput.stream_);
+ // first layer, conv, use stride=2
+ ConvForward(ninput, Ki2h, nhidden, ksize, kstride, tmp_col, tmp_dst);
+ // add bias
+ nhidden += broadcast<1>(hbias, nhidden.shape_);
+ // activation, relu, backup activation in nhidden
+ nhidden = F<relu>(nhidden);
+ Copy(nhiddenbak, nhidden, nhiddenbak.stream_);
+ // max pooling
+ npool = pool<red::maximum>(nhiddenbak, npool[0][0].shape_, psize, psize, psize);
+ Copy(npoolbak, npool, npoolbak.stream_);
+ // flat
+ nflat = reshape(npool, nflat.shape_);
+ // second layer fullc
+ nout = dot(nflat, Wh2o);
+ nout += repmat(obias, batch_size);
+ // softmax calculation
+ Softmax(nout, nout);
+ // copy result out
+ Copy(oubatch, nout, nout.stream_);
+ }
+ // back propagation
+ virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) {
+ // copy gradient to output layer
+ Copy(nout, gradout, nout.stream_);
+ // calc grad of final layer
+ g_obias = sum_rows(nout);
+ g_Wh2o = dot(nflat.T(), nout);
+ // backprop to previous layer
+ nflat = dot(nout, Wh2o.T());
+ npool = reshape(nflat, npool.shape_);
+ // backprop pooling layer
+ nhiddenbak = unpool<red::maximum>(nhiddenbak, npoolbak, npool, psize, psize, psize);
+ // calculate gradient of relu layer
+ nhidden = F<relu_grad>(nhidden) * nhiddenbak;
+ // calc grad of layer 1
+ g_hbias = sumall_except_dim<1>(nhidden);
+ ConvBackWard(nhidden, Ki2h, g_Ki2h, ninput, ksize, kstride, tmp_col, tmp_dst);
+ }
+ // update weight
+ virtual void Update(void) {
+ // run SGD
+ const float eta = 0.1;
+ const float wd = 0.00001;
+ // update weight
+ Ki2h -= eta * (wd * Ki2h + g_Ki2h);
+ Wh2o -= eta * (wd * Wh2o + g_Wh2o);
+ // no regularization for bias
+ hbias-= eta * g_hbias;
+ obias-= eta * g_obias;
+ }
+ private:
+ // forward convolution, tmp_col and tmp_dst are helper structure
+ inline static void ConvForward(const Tensor<xpu, 4, real_t> &in,
+ const Tensor<xpu, 2, real_t> &kernel,
+ Tensor<xpu, 4, real_t> &out,
+ int ksize, int kstride,
+ TensorContainer<xpu, 2, real_t> &tmp_col,
+ TensorContainer<xpu, 2, real_t> &tmp_dst) {
+ index_t oheight = (in.size(2) - ksize)/kstride + 1;
+ index_t owidth = (in.size(3) - ksize)/kstride + 1;
+ index_t nbatch = in.size(0);
+ index_t nchannel = out.size(1);
+ // we directly unpack all local patches and do a dot product
+ // this cost lots of memory, normally for large image, only unpack several image at a time
+ tmp_col.Resize(Shape2(in.size(1)*ksize*ksize, nbatch*oheight*owidth));
+ tmp_dst.Resize(Shape2(nchannel, nbatch*oheight*owidth));
+ // unpack local patches , stride=1
+ tmp_col = unpack_patch2col(in, ksize, ksize, kstride, kstride, 1, 1);
+ tmp_dst = dot(kernel, tmp_col);
+ // reshape, then swap axis, we chain equations together
+ out = swapaxis<1,0>(reshape(tmp_dst, Shape4(nchannel, nbatch, oheight, owidth)));
+ }
+ // backward convolution, calculate gradient of kernel, and backprop back to in
+ inline static void ConvBackWard(const Tensor<xpu, 4, real_t> &out,
+ const Tensor<xpu, 2, real_t> &kernel,
+ Tensor<xpu, 2, real_t> &g_kernel,
+ Tensor<xpu, 4, real_t> &in,
+ int ksize, int kstride,
+ TensorContainer<xpu, 2, real_t> &tmp_col,
+ TensorContainer<xpu, 2, real_t> &tmp_dst) {
+ index_t oheight = (in.size(2) - ksize)/kstride + 1;
+ index_t owidth = (in.size(3) - ksize)/kstride + 1;
+ index_t nbatch = in.size(0);
+ index_t nchannel = out.size(1);
+ // we directly unpack all local patches and do a dot product
+ // this cost lots of memory, normally for large image, only unpack several image at a time
+ tmp_col.Resize(Shape2(in.size(1) * ksize * ksize,
+ nbatch * oheight * owidth));
+ tmp_dst.Resize(Shape2(nchannel, nbatch * oheight * owidth));
+ // unpack local patches
+ tmp_col = unpack_patch2col(in, ksize, ksize, kstride, kstride, 1, 1);
+ tmp_dst = reshape(swapaxis<1,0>(out), tmp_dst.shape_);
+ g_kernel = dot(tmp_dst, tmp_col.T());
+ // backpropgation: not necessary for first layer, but included anyway
+ tmp_col = dot(kernel.T(), tmp_dst);
+ in = pack_col2patch(tmp_col, in.shape_, ksize, ksize, kstride, kstride, 1, 1);
+ }
+ private:
+ // random seed generator
+ Random<xpu, real_t> rnd;
+ // kernel size, pooling size
+ int ksize, kstride, psize;
+ // nodes in neural net
+ TensorContainer<xpu, 4, real_t> ninput, nhidden, nhiddenbak, npool, npoolbak;
+ TensorContainer<xpu, 2, real_t> nflat, nout;
+ // temp helper structure
+ TensorContainer<xpu, 2, real_t> tmp_col, tmp_dst;
+ // hidden bias, gradient
+ TensorContainer<xpu, 1, real_t> hbias, obias, g_hbias, g_obias;
+ // weight, gradient: Ki2h is actually convoltuion kernel, with shape=(num_channel,ksize*ksize)
+ TensorContainer<xpu, 2, real_t> Ki2h, Wh2o, g_Ki2h, g_Wh2o;
+};
+
+// helper function to get the max inde
+inline int MaxIndex(Tensor<cpu, 1, real_t> pred) {
+ int maxidx = 0;
+ for (index_t i = 1; i < pred.size(0); ++i) {
+ if(pred[i] > pred[maxidx]) maxidx = (int)i;
+ }
+ return maxidx;
+}
+
+int main(int argc, char *argv[]) {
+ if(argc < 2) {
+ printf("Usage: cpu or gpu\n"); return 0;
+ }
+ srand(0);
+ // settings
+ int batch_size = 100;
+ int insize = 28;
+ int nchannel = 10;
+ int ksize = 5;
+ int kstride = 1;
+ int psize = 2;
+ int num_out = 10;
+
+ // choose which version to use
+ INNet *net;
+ if (!strcmp(argv[1], "gpu")) {
+ InitTensorEngine<gpu>();
+ net = new ConvNet<gpu>(batch_size, insize, nchannel, ksize, kstride, psize, num_out);
+ } else {
+ InitTensorEngine<cpu>();
+ net = new ConvNet<cpu>(batch_size, insize, nchannel, ksize, kstride, psize, num_out);
+ }
+
+ // temp output layer
+ TensorContainer<cpu, 2, real_t> pred;
+ pred.Resize(Shape2(batch_size, num_out));
+
+ // label
+ std::vector<int> ytrain, ytest;
+ // data
+ TensorContainer<cpu, 2, real_t> xtrain_, xtest_;
+ LoadMNIST("train-images-idx3-ubyte", "train-labels-idx1-ubyte", ytrain, xtrain_, true);
+ LoadMNIST("t10k-images-idx3-ubyte", "t10k-labels-idx1-ubyte", ytest, xtest_, false);
+
+ TensorContainer<cpu, 4, real_t> xtrain(Shape4(xtrain_.size(0), 1, insize, insize));
+ TensorContainer<cpu, 4, real_t> xtest(Shape4(xtest_.size(0), 1, insize, insize));
+ xtrain = reshape(xtrain_, xtrain.shape_);
+ xtest = reshape(xtest_, xtest.shape_);
+
+ int num_iter = 20;
+
+ for (int i = 0; i < num_iter; ++ i) {
+ // training
+ for (index_t j = 0; j + batch_size <= xtrain.size(0); j += batch_size) {
+ net->Forward(xtrain.Slice(j, j + batch_size), pred);
+ // set gradient into pred
+ for (int k = 0; k < batch_size; ++ k) {
+ pred[k][ ytrain[k+j] ] -= 1.0f;
+ }
+ // scale gradient by batchs zie
+ pred *= 1.0f / batch_size;
+ // run backprop
+ net->Backprop(pred);
+ // update net parameters
+ net->Update();
+ }
+ // evaluation
+ long nerr = 0;
+ for (index_t j = 0; j + batch_size <= xtest.size(0); j += batch_size) {
+ net->Forward(xtest.Slice(j, j + batch_size), pred);
+ for (int k = 0; k < batch_size; ++ k) {
+ nerr += MaxIndex(pred[k]) != ytest[j+k];
+ }
+ }
+ printf("round %d: test-err=%f\n", i, (float)nerr/xtest.size(0));
+ }
+ delete net;
+
+ if (!strcmp(argv[1], "gpu")) {
+ ShutdownTensorEngine<gpu>();
+ } else {
+ ShutdownTensorEngine<cpu>();
+ }
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/neuralnet/nnet.cu b/3rdparty/mshadow/guide/neuralnet/nnet.cu
new file mode 100644
index 0000000..6ef8b0d
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/nnet.cu
@@ -0,0 +1,202 @@
+// this implements a simple two layer neural net
+#include <vector>
+#include <cmath>
+// header file to use mshadow
+#include "mshadow/tensor.h"
+// helper function to load mnist dataset
+#include "util.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+// define sigmoid operation
+struct sigmoid{
+ MSHADOW_XINLINE static real_t Map(real_t a) {
+ return 1.0f/(1.0f+expf(-a));
+ }
+};
+
+/*! \brief interface for nnet, interfacd allows use to use GPU/CPU implementation in a unified way */
+class INNet{
+ public:
+ virtual void Forward(const Tensor<cpu, 2, real_t>& inbatch, Tensor<cpu, 2, real_t> &oubatch) = 0;
+ virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) = 0;
+ virtual void Update(void) = 0;
+ virtual ~INNet() {}
+};
+
+/*!
+ * \brief simple two layer neural net
+ * this implementation is device invariant
+ */
+template<typename xpu>
+class NNet : public INNet {
+ public:
+ // initialize the network
+ NNet(int batch_size, int num_in, int num_hidden, int num_out) : rnd(0) {
+ // setup stream
+ Stream<xpu> *stream = NewStream<xpu>();
+ ninput.set_stream(stream);
+ nhidden.set_stream(stream);
+ nhiddenbak.set_stream(stream);
+ nout.set_stream(stream);
+ hbias.set_stream(stream);
+ g_hbias.set_stream(stream);
+ g_obias.set_stream(stream);
+ obias.set_stream(stream);
+ Wi2h.set_stream(stream);
+ Wh2o.set_stream(stream);
+ g_Wi2h.set_stream(stream);
+ g_Wh2o.set_stream(stream);
+ // setup nodes
+ ninput.Resize(Shape2(batch_size, num_in));
+ nhidden.Resize(Shape2(batch_size, num_hidden));
+ nhiddenbak.Resize(nhidden.shape_);
+ nout.Resize(Shape2(batch_size, num_out));
+ // setup bias
+ hbias.Resize(Shape1(num_hidden)); g_hbias.Resize(hbias.shape_);
+ obias.Resize(Shape1(num_out)); g_obias.Resize(obias.shape_);
+ hbias = 0.0f; obias = 0.0f;
+ // setup weights
+ Wi2h.Resize(Shape2(num_in, num_hidden)); g_Wi2h.Resize(Wi2h.shape_);
+ Wh2o.Resize(Shape2(num_hidden, num_out)); g_Wh2o.Resize(Wh2o.shape_);
+ rnd.SampleGaussian(&Wi2h, 0, 0.01f);
+ rnd.SampleGaussian(&Wh2o, 0, 0.01f);
+ }
+ virtual ~NNet() {}
+ // forward propagation
+ virtual void Forward(const Tensor<cpu, 2, real_t>& inbatch,
+ Tensor<cpu, 2, real_t> &oubatch) {
+ // size is same conventsion as numpy
+ index_t batch_size = inbatch.size(0);
+ // copy data to input layer
+ Copy(ninput, inbatch, ninput.stream_);
+ // first layer, fullc
+ nhidden = dot(ninput, Wi2h);
+ nhidden+= repmat(hbias, batch_size);
+ // activation, sigmloid, backup activation in nhidden
+ nhidden = F<sigmoid>(nhidden);
+ Copy(nhiddenbak, nhidden, nhiddenbak.stream_);
+ // second layer fullc
+ nout = dot(nhiddenbak, Wh2o);
+ nout += repmat(obias, batch_size);
+ // softmax calculation
+ Softmax(nout, nout);
+ // copy result out
+ Copy(oubatch, nout, nout.stream_);
+ }
+ // back propagation
+ virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) {
+ // copy gradient to output layer
+ Copy(nout, gradout, nout.stream_);
+ // calc grad of layer 2
+ g_obias = sum_rows(nout);
+ g_Wh2o = dot(nhiddenbak.T(), nout);
+ // backprop to layer 1
+ nhiddenbak = dot(nout, Wh2o.T());
+ // calculate gradient of sigmoid layer
+ nhidden = nhidden * (1.0f-nhidden) * nhiddenbak;
+ // calc grad of layer 1
+ g_hbias = sum_rows(nhidden);
+ g_Wi2h = dot(ninput.T(), nhidden);
+ }
+ // update weight
+ virtual void Update(void) {
+ // run SGD
+ const float eta = 0.8;
+ const float wd = 0.00001;
+ // update weight
+ Wi2h -= eta * (wd * Wi2h + g_Wi2h);
+ Wh2o -= eta * (wd * Wh2o + g_Wh2o);
+ // no regularization for bias
+ hbias-= eta * g_hbias;
+ obias-= eta * g_obias;
+ }
+ private:
+ // random seed generator
+ Random<xpu, real_t> rnd;
+ // nodes in neural net
+ TensorContainer<xpu, 2, real_t> ninput, nhidden, nhiddenbak, nout;
+ // hidden bias, gradient
+ TensorContainer<xpu, 1, real_t> hbias, obias, g_hbias, g_obias;
+ // weight gradient
+ TensorContainer<xpu, 2, real_t> Wi2h, Wh2o, g_Wi2h, g_Wh2o;
+};
+// helper function to get the max inde
+inline int MaxIndex(Tensor<cpu, 1, real_t> pred) {
+ int maxidx = 0;
+ for(index_t i = 1; i < pred.size(0); ++i) {
+ if(pred[i] > pred[maxidx]) maxidx = (int)i;
+ }
+ return maxidx;
+}
+
+int main(int argc, char *argv[]) {
+ if(argc < 2) {
+ printf("Usage: cpu or gpu\n"); return 0;
+ }
+ srand(0);
+
+ // settings
+ int batch_size = 100;
+ int num_in = 28 * 28;
+ int num_hidden = 100;
+ int num_out = 10;
+ // choose which version to use
+ INNet *net;
+ if (!strcmp(argv[1], "gpu")) {
+ InitTensorEngine<gpu>();
+ net = new NNet<gpu>(batch_size, num_in, num_hidden, num_out);
+ } else {
+ InitTensorEngine<cpu>();
+ net = new NNet<cpu>(batch_size, num_in, num_hidden, num_out);
+ }
+
+ // temp output layer
+ TensorContainer<cpu, 2, real_t> pred;
+ pred.Resize(Shape2(batch_size, num_out));
+
+ // label
+ std::vector<int> ytrain, ytest;
+ // data
+ TensorContainer<cpu,2> xtrain, xtest;
+ LoadMNIST("train-images-idx3-ubyte", "train-labels-idx1-ubyte", ytrain, xtrain, true);
+ LoadMNIST("t10k-images-idx3-ubyte", "t10k-labels-idx1-ubyte", ytest, xtest, false);
+
+ int num_iter = 20;
+
+ for (int i = 0; i < num_iter; ++ i) {
+ // training
+ for (index_t j = 0; j + batch_size <= xtrain.size(0); j += batch_size) {
+ net->Forward(xtrain.Slice(j, j + batch_size), pred);
+ // set gradient into pred
+ for (int k = 0; k < batch_size; ++ k) {
+ pred[k][ ytrain[k+j] ] -= 1.0f;
+ }
+ // scale gradient by batchs zie
+ pred *= 1.0f / batch_size;
+ // run backprop
+ net->Backprop(pred);
+ // update net parameters
+ net->Update();
+ }
+ // evaluation
+ long nerr = 0;
+ for (index_t j = 0; j + batch_size <= xtest.size(0); j += batch_size) {
+ net->Forward(xtest.Slice(j, j + batch_size), pred);
+ for (int k = 0; k < batch_size; ++ k) {
+ nerr += MaxIndex(pred[k]) != ytest[j+k];
+
+ }
+ }
+ printf("round %d: test-err=%f\n", i, (float)nerr/xtest.size(0));
+ }
+ delete net;
+ if (!strcmp(argv[1], "gpu")) {
+ ShutdownTensorEngine<gpu>();
+ } else {
+ ShutdownTensorEngine<cpu>();
+ }
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/neuralnet/nnet_ps.cu b/3rdparty/mshadow/guide/neuralnet/nnet_ps.cu
new file mode 100644
index 0000000..5c03fb1
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/nnet_ps.cu
@@ -0,0 +1,312 @@
+// this implements a simple two layer Multi-GPU neural net
+// this implementation uses mshadow-ps to get gradient aggregation
+// between cards
+// this code is modified from nnet.cu
+#include <vector>
+#include <cmath>
+#include <omp.h>
+// header file to use mshadow
+#include <mshadow/tensor.h>
+#include <mshadow-ps/mshadow_ps.h>
+// helper function to load mnist dataset
+#include "./util.h"
+// this namespace contains all data structures, functions
+using namespace mshadow;
+// this namespace contains all operator overloads
+using namespace mshadow::expr;
+
+// define sigmoid operation
+struct sigmoid {
+ MSHADOW_XINLINE static real_t Map(real_t a) {
+ return 1.0f / (1.0f + expf(-a));
+ }
+};
+
+/*! \brief interface for nnet, interfacd allows use to use GPU/CPU implementation in a unified way */
+class INNet{
+ public:
+ virtual void Forward(const Tensor<cpu, 2, real_t>& inbatch,
+ Tensor<cpu, 2, real_t> &oubatch) = 0;
+ virtual void Backprop(const Tensor<cpu, 2, real_t>& gradout) = 0;
+ virtual ~INNet() {}
+};
+
+/*!
+ * \brief simple two layer neural net
+ * this implementation is device invariant
+ */
+template<typename xpu>
+class NNet : public INNet {
+ public:
+ // initialize the network
+ NNet(int batch_size, int num_in, int num_hidden, int num_out,
+ int devid, mshadow::ps::ISharedModel<xpu, real_t> *ps)
+ : rnd(0), devid(devid), ps(ps) {
+ mshadow::SetDevice<xpu>(devid);
+ stream = mshadow::NewStream<xpu>();
+ // set the computing streams
+ ninput.set_stream(stream);
+ nhidden.set_stream(stream);
+ nhiddenbak.set_stream(stream);
+ nout.set_stream(stream);
+ hbias.set_stream(stream);
+ obias.set_stream(stream);
+ g_hbias.set_stream(stream);
+ g_obias.set_stream(stream);
+ Wi2h.set_stream(stream);
+ Wh2o.set_stream(stream);
+ g_Wi2h.set_stream(stream);
+ g_Wh2o.set_stream(stream);
+ rnd.set_stream(stream);
+ // setup nodes
+ ninput.Resize(Shape2(batch_size, num_in));
+ nhidden.Resize(Shape2(batch_size, num_hidden));
+ nhiddenbak.Resize(nhidden.shape_);
+ nout.Resize(Shape2(batch_size, num_out));
+ // setup bias
+ hbias.Resize(Shape1(num_hidden)); g_hbias.Resize(hbias.shape_);
+ obias.Resize(Shape1(num_out)); g_obias.Resize(obias.shape_);
+ hbias = 0.0f; obias = 0.0f;
+ // setup weights
+ Wi2h.Resize(Shape2(num_in, num_hidden)); g_Wi2h.Resize(Wi2h.shape_);
+ Wh2o.Resize(Shape2(num_hidden, num_out)); g_Wh2o.Resize(Wh2o.shape_);
+ rnd.SampleGaussian(&Wi2h, 0, 0.01f);
+ rnd.SampleGaussian(&Wh2o, 0, 0.01f);
+ // initialize the key
+ ps->InitKey(Wi2h.shape_, 0, devid);
+ ps->InitKey(hbias.shape_, 1, devid);
+ ps->InitKey(Wh2o.shape_, 2, devid);
+ ps->InitKey(obias.shape_, 3, devid);
+ }
+ virtual ~NNet() {
+ mshadow::SetDevice<xpu>(devid);
+ mshadow::DeleteStream(stream);
+ }
+ // forward propagation
+ virtual void Forward(const Tensor<cpu, 2, real_t> &inbatch,
+ Tensor<cpu, 2, real_t> &oubatch) {
+ // size is same conventsion as numpy
+ index_t batch_size = inbatch.size(0);
+ // copy data to input layer
+ Copy(ninput, inbatch, stream);
+ // wait the last pull requst on layer to complete
+ ps->PullWait(0, devid);
+ // first layer, fullc
+ nhidden = dot(ninput, Wi2h);
+ // wait the pull request on hbias to complete
+ ps->PullWait(1, devid);
+ nhidden+= repmat(hbias, batch_size);
+ // activation, sigmloid, backup activation in nhidden
+ nhidden = F<sigmoid>(nhidden);
+ Copy(nhiddenbak, nhidden, stream);
+ // second layer fullc
+ ps->PullWait(2, devid);
+ nout = dot(nhiddenbak, Wh2o);
+ ps->PullWait(3, devid);
+ nout += repmat(obias, batch_size);
+ // softmax calculation
+ Softmax(nout, nout);
+ // copy result out
+ Copy(oubatch, nout, stream);
+ // Copy with stream is non-blocking, use wait to wait until copy finishes
+ stream->Wait();
+ }
+ // back propagation
+ virtual void Backprop(const Tensor<cpu, 2, real_t> &gradout) {
+ // copy gradient to output layer
+ Copy(nout, gradout, stream);
+ // calc grad of layer 2
+ g_obias = sum_rows(nout);
+ // sync proc defines the synchronization step
+ this->SyncProc(obias, g_obias, 3);
+ // update second layer weights
+ g_Wh2o = dot(nhiddenbak.T(), nout);
+ // backprop to layer 1
+ nhiddenbak = dot(nout, Wh2o.T());
+ this->SyncProc(Wh2o, g_Wh2o, 2);
+ // calculate gradient of sigmoid layer
+ nhidden = nhidden * (1.0f-nhidden) * nhiddenbak;
+ // calc grad of layer 1
+ g_hbias = sum_rows(nhidden);
+ this->SyncProc(hbias, g_hbias, 1);
+ g_Wi2h = dot(ninput.T(), nhidden);
+ this->SyncProc(Wi2h, g_Wi2h, 0);
+ }
+ // synchronization function
+ template<int dim>
+ inline void SyncProc(mshadow::Tensor<xpu, dim> weight,
+ mshadow::Tensor<xpu, dim> grad,
+ int data_key) {
+ // wait till last computation finishes
+ stream->Wait();
+ ps->Push(grad, data_key, devid, -data_key);
+ ps->PullReq(grad, data_key, devid, -data_key,
+ UpdateEntry::ApplyUpdate,
+ new UpdateEntry(weight.FlatTo2D(), grad.FlatTo2D(), dim == 1));
+ }
+ // data structure defined to help using callback function
+ struct UpdateEntry {
+ mshadow::Tensor<xpu, 2> weight;
+ mshadow::Tensor<xpu, 2> grad;
+ bool is_bias;
+ // constructor
+ UpdateEntry(mshadow::Tensor<xpu, 2> weight,
+ mshadow::Tensor<xpu, 2> grad,
+ bool is_bias)
+ : weight(weight), grad(grad),
+ is_bias(is_bias) {}
+ inline void Update(mshadow::Stream<xpu> *stream) {
+ weight.set_stream(stream);
+ const float wd = 0.00001;
+ const float eta = 0.8;
+ if (!is_bias) {
+ weight -= eta * (wd * weight + grad);
+ } else {
+ weight -= eta * grad;
+ }
+ }
+ // callback function to apply update
+ inline static void ApplyUpdate(mshadow::Stream<xpu> *stream, void *arg) {
+ UpdateEntry *e = static_cast<UpdateEntry*>(arg);
+ e->Update(stream);
+ delete e;
+ }
+ };
+
+ private:
+ // computing stream
+ mshadow::Stream<xpu> *stream;
+ // device id
+ int devid;
+ // parameter server interface
+ mshadow::ps::ISharedModel<xpu, real_t> *ps;
+ // random seed generator
+ Random<xpu, real_t> rnd;
+ // nodes in neural net
+ TensorContainer<xpu, 2, real_t> ninput, nhidden, nhiddenbak, nout;
+ // hidden bias, gradient
+ TensorContainer<xpu, 1, real_t> hbias, obias, g_hbias, g_obias;
+ // weight gradient
+ TensorContainer<xpu, 2, real_t> Wi2h, Wh2o, g_Wi2h, g_Wh2o;
+};
+
+// helper function to get the max inde
+inline int MaxIndex(Tensor<cpu, 1, real_t> pred) {
+ int maxidx = 0;
+ for(index_t i = 1; i < pred.size(0); ++i) {
+ if(pred[i] > pred[maxidx]) maxidx = (int)i;
+ }
+ return maxidx;
+}
+
+namespace mshadow {
+namespace ps {
+// model updater is used when update is happening on server side
+// if we only use parameter server for sum aggregation
+// this is not needed, but we must declare this function to return NULL
+template<>
+IModelUpdater<float> *CreateModelUpdater(void) {
+ return NULL;
+}
+}
+}
+
+template<typename xpu>
+inline int Run(int argc, char *argv[]) {
+ srand(0);
+ // settings
+ int batch_size = 100;
+ int num_in = 28 * 28;
+ int num_hidden = 100;
+ int num_out = 10;
+ int ndev = argc - 2;
+ if (batch_size % ndev != 0) {
+ fprintf(stderr, "choose number of devices ndev such that 100 MOD ndev == 0\n");
+ return 0;
+ }
+ // choose which version to use
+ std::vector<int> devs;
+ for (int i = 2; i < argc; ++i) {
+ devs.push_back(atoi(argv[i]));
+ }
+ mshadow::ps::ISharedModel<xpu, real_t>
+ *ps = mshadow::ps::CreateSharedModel<xpu, real_t>("local");
+ ps->Init(devs);
+
+ std::vector<INNet *> nets(ndev);
+ for (int i = 0; i < ndev; ++i) {
+ mshadow::InitTensorEngine<xpu>(devs[i]);
+ nets[i] = new NNet<xpu>(batch_size / ndev, num_in, num_hidden, num_out, devs[i], ps);
+ }
+
+ // label
+ std::vector<int> ytrain, ytest;
+ // data
+ TensorContainer<cpu,2> xtrain, xtest;
+ LoadMNIST("train-images-idx3-ubyte", "train-labels-idx1-ubyte", ytrain, xtrain, true);
+ LoadMNIST("t10k-images-idx3-ubyte", "t10k-labels-idx1-ubyte", ytest, xtest, false);
+ int num_iter = 20;
+
+ for (int i = 0; i < num_iter; ++ i) {
+ // mini-batch per device
+ int step = batch_size / ndev;
+ // running parallel threads
+ #pragma omp parallel num_threads(ndev)
+ {
+ // temp output layer
+ TensorContainer<cpu, 2, real_t> pred;
+ pred.Resize(Shape2(step, num_out));
+ int tid = omp_get_thread_num();
+ mshadow::SetDevice<xpu>(devs[tid]);
+ for (index_t j = 0; j + batch_size <= xtrain.size(0); j += batch_size) {
+ nets[tid]->Forward(xtrain.Slice(j + tid * step, j + (tid + 1) * step), pred);
+ // set gradient into pred
+ for (int k = 0; k < step; ++ k) {
+ pred[k][ytrain[j + tid * step + k]] -= 1.0f;
+ }
+ // scale gradient by batchs zie
+ pred *= 1.0f / batch_size;
+ // run backprop
+ nets[tid]->Backprop(pred);
+ }
+ }
+ // evaluation
+ long nerr = 0;
+ #pragma omp parallel num_threads(ndev) reduction(+:nerr)
+ {
+ // temp output layer
+ TensorContainer<cpu, 2, real_t> pred;
+ pred.Resize(Shape2(step, num_out));
+ int tid = omp_get_thread_num();
+ mshadow::SetDevice<xpu>(devs[tid]);
+ for (index_t j = 0; j + batch_size <= xtest.size(0); j += batch_size) {
+ nets[tid]->Forward(xtest.Slice(j + tid * step, j + (tid + 1) * step), pred);
+ for (int k = 0; k < step; ++ k) {
+ nerr += MaxIndex(pred[k]) != ytest[j + tid * step + k];
+ }
+ }
+ }
+ printf("round %d: test-err=%f\n", i, (float)nerr/xtest.size(0));
+ }
+
+ for(int i = 0; i < ndev; ++i) {
+ mshadow::SetDevice<xpu>(devs[i]);
+ delete nets[i];
+ ShutdownTensorEngine<xpu>();
+ }
+ return 0;
+}
+int main(int argc, char *argv[]) {
+ if (argc < 3) {
+ printf("Usage: <device> devicelist\n"\
+ "\tExample1: ./nnet_ps cpu 1 2 3\n"\
+ "\tExample2: ./nnet_ps gpu 0 1\n");
+ return 0;
+ }
+ if (!strcmp(argv[1], "cpu")) {
+ Run<mshadow::cpu>(argc, argv);
+ } else {
+ Run<mshadow::gpu>(argc, argv);
+ }
+ return 0;
+}
diff --git a/3rdparty/mshadow/guide/neuralnet/util.h b/3rdparty/mshadow/guide/neuralnet/util.h
new file mode 100644
index 0000000..f58203c
--- /dev/null
+++ b/3rdparty/mshadow/guide/neuralnet/util.h
@@ -0,0 +1,86 @@
+#pragma once
+#include <assert.h>
+#include <cstdio>
+#include <cstdlib>
+#include "mshadow/tensor.h"
+
+typedef float real_t;
+
+using namespace mshadow;
+
+int pack(unsigned char zz[4]){
+ return (int)(zz[3])
+ | (((int)(zz[2])) << 8)
+ | (((int)(zz[1])) << 16)
+ | (((int)(zz[0])) << 24);
+}
+
+template<typename T>
+inline void shuffle(T *data, size_t sz){
+ if(sz == 0) return;
+ for(size_t i = sz - 1; i > 0; i--){
+ std::swap(data[i], data[rand() % (i+1)]);
+ }
+}
+// random shuffle the data inside, require PRNG
+template<typename T>
+inline void shuffle(std::vector<T> &data){
+ shuffle(&data[0], data.size());
+}
+
+// simple function to load in mnist
+inline void LoadMNIST(const char *path_img, const char *path_label,
+ std::vector<int> &ylabel,
+ TensorContainer<cpu, 2, real_t> &xdata,
+ bool do_shuffle){
+ // load in data
+ FILE *fi = fopen(path_img, "rb");
+ if (fi == NULL) {
+ printf("cannot open %s\n", path_img);
+ exit(-1);
+ }
+ unsigned char zz[4];
+ unsigned char *t_data, *l_data;
+ int num_image, width, height, nlabel;
+ assert(fread(zz, 4 , 1, fi));
+ assert(fread(zz, 4 , 1, fi));
+ num_image = pack(zz);
+ assert(fread(zz, 4 , 1, fi));
+ width = pack(zz);
+ assert(fread(zz, 4 , 1, fi));
+ height = pack(zz);
+
+ int step = width * height;
+ t_data = new unsigned char[num_image * step];
+ assert(fread(t_data, step*num_image , 1 , fi));
+ fclose(fi);
+
+ // load in label
+ fi = fopen(path_label, "rb");
+ assert(fread(zz, 4 , 1, fi));
+ assert(fread(zz, 4 , 1, fi));
+ nlabel = pack(zz);
+ assert(num_image == nlabel);
+ l_data = new unsigned char[num_image];
+ assert(fread(l_data, num_image , 1 , fi));
+ // try to do shuffle
+ std::vector<int> rindex;
+ for (int i = 0; i < num_image; ++ i) {
+ rindex.push_back(i);
+ }
+ if (do_shuffle) {
+ shuffle(rindex);
+ }
+
+ // save out result
+ ylabel.resize(num_image);
+ xdata.Resize(Shape2(num_image, width * height));
+ for (int i = 0 ; i < num_image ; ++i) {
+ for(int j = 0; j < step; ++j) {
+ xdata[i][j] = (float)(t_data[rindex[i]*step + j]) / 256.0f;
+ }
+ ylabel[i] = l_data[rindex[i]];
+ }
+ delete[] t_data; delete [] l_data;
+ printf("finish loading %dx%d matrix from %s, shuffle=%d\n", num_image, step, path_img, (int)do_shuffle);
+}
diff --git a/3rdparty/mshadow/make/README.md b/3rdparty/mshadow/make/README.md
new file mode 100644
index 0000000..6ef24d6
--- /dev/null
+++ b/3rdparty/mshadow/make/README.md
@@ -0,0 +1,18 @@
+Makefile Configuration of MShadow
+=====
+MShadow is a template library, you only need to include mshadow to use it. So this folder is not used to build mshadow library file.
+
+However, mshadow is a flexible library that allows you to compile with different configurations. For example,
+you can compile mshadow without CUDA, and specify your own choice of BLAS.
+There are different compile flags that you might need to set in your own configuration.
+This folder provides a Makefile script to help you do that.
+
+Usage
+=====
+* Set the configurations via variables in your Makefile, see example in [../guide/config.mk](../guide/config.mk)
+* include [mshadow.mk](mshadow.mk) in your Makefile
+* mshadow.mk will give you compiler variables that you can include when compiling
+ - Add MSHADOW_CFLAGS to the compile flags
+ - Add MSHADOW_LDFLAGS to the linker flags
+ - Add MSHADOW_NVCCFLAGS to the nvcc compile flags
+* For example Makefile, see [../guide/Makefile](../guide/Makefile)
diff --git a/3rdparty/mshadow/make/mshadow.mk b/3rdparty/mshadow/make/mshadow.mk
new file mode 100644
index 0000000..86155ea
--- /dev/null
+++ b/3rdparty/mshadow/make/mshadow.mk
@@ -0,0 +1,166 @@
+#---------------------------------------------------------------------------------------
+# mshadow configuration script
+#
+# include mshadow.mk after the variables are set
+#
+# Add MSHADOW_CFLAGS to the compile flags
+# Add MSHADOW_LDFLAGS to the linker flags
+# Add MSHADOW_NVCCFLAGS to the nvcc compile flags
+#----------------------------------------------------------------------------------------
+
+MSHADOW_CFLAGS = -funroll-loops -Wno-unused-parameter -Wno-unknown-pragmas -Wno-unused-local-typedefs
+MSHADOW_LDFLAGS = -lm
+MSHADOW_NVCCFLAGS =
+
+
+# atlas blas library has different name on CentOS
+OS := $(shell cat /etc/system-release 2>/dev/null)
+ifeq ($(findstring CentOS,$(OS)), CentOS)
+ ATLAS_LDFLAGS := -lsatlas -L/usr/lib64/atlas
+else
+ ATLAS_LDFLAGS := -lcblas
+endif
+
+ifndef USE_SSE
+ USE_SSE=1
+endif
+
+ifeq ($(USE_SSE), 1)
+ MSHADOW_CFLAGS += -msse3
+else
+ MSHADOW_CFLAGS += -DMSHADOW_USE_SSE=0
+endif
+
+# whether to use F16C instruction set extension for fast fp16 compute on CPU
+# if cross compiling you may want to explicitly turn it off if target system does not support it
+ifndef USE_F16C
+ ifneq ($(OS),Windows_NT)
+ detected_OS := $(shell uname -s)
+ ifeq ($(detected_OS),Darwin)
+ F16C_SUPP = $(shell sysctl -a | grep machdep.cpu.features | grep F16C)
+ endif
+ ifeq ($(detected_OS),Linux)
+ F16C_SUPP = $(shell cat /proc/cpuinfo | grep flags | grep f16c)
+ endif
+ ifneq ($(strip $(F16C_SUPP)),)
+ USE_F16C=1
+ else
+ USE_F16C=0
+ endif
+ endif
+ # if OS is Windows, check if your processor and compiler support F16C architecture.
+ # One way to check if processor supports it is to download the tool
+ # https://docs.microsoft.com/en-us/sysinternals/downloads/coreinfo.
+ # If coreinfo -c shows F16C and compiler supports it,
+ # then you can set USE_F16C=1 explicitly to leverage that capability"
+endif
+
+ifeq ($(USE_F16C), 1)
+ MSHADOW_CFLAGS += -mf16c
+else
+ MSHADOW_CFLAGS += -DMSHADOW_USE_F16C=0
+endif
+
+ifeq ($(USE_CUDA), 0)
+ MSHADOW_CFLAGS += -DMSHADOW_USE_CUDA=0
+else
+ MSHADOW_LDFLAGS += -lcudart -lcublas -lcurand -lcusolver
+endif
+ifneq ($(USE_CUDA_PATH), NONE)
+ MSHADOW_CFLAGS += -I$(USE_CUDA_PATH)/include
+ MSHADOW_LDFLAGS += -L$(USE_CUDA_PATH)/lib64 -L$(USE_CUDA_PATH)/lib
+endif
+
+ifeq ($(USE_BLAS), mkl)
+ifneq ($(USE_INTEL_PATH), NONE)
+ UNAME_S := $(shell uname -s)
+ ifeq ($(UNAME_S),Darwin)
+ MSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/mkl/lib
+ MSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/lib
+ else
+ MSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/mkl/lib/intel64
+ MSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/compiler/lib/intel64
+ MSHADOW_LDFLAGS += -L$(USE_INTEL_PATH)/lib/intel64
+ endif
+ MSHADOW_CFLAGS += -I$(USE_INTEL_PATH)/mkl/include
+endif
+ifneq ($(USE_STATIC_MKL), NONE)
+ifeq ($(USE_INTEL_PATH), NONE)
+ MKLROOT = /opt/intel/mkl
+else
+ MKLROOT = $(USE_INTEL_PATH)/mkl
+endif
+ MSHADOW_LDFLAGS += -L${MKLROOT}/../compiler/lib/intel64 -Wl,--start-group ${MKLROOT}/lib/intel64/libmkl_intel_lp64.a ${MKLROOT}/lib/intel64/libmkl_core.a ${MKLROOT}/lib/intel64/libmkl_intel_thread.a -Wl,--end-group -liomp5 -ldl -lpthread -lm
+else
+ifneq ($(USE_MKLML), 1)
+ MSHADOW_LDFLAGS += -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5
+endif
+endif
+else
+ifneq ($(USE_BLAS), NONE)
+ MSHADOW_CFLAGS += -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
+endif
+endif
+
+ifeq ($(USE_MKLML), 1)
+ MSHADOW_CFLAGS += -I$(MKLROOT)/include
+ ifneq ($(shell uname),Darwin)
+ MSHADOW_LDFLAGS += -Wl,--as-needed -lmklml_intel -lmklml_gnu
+ else
+ MSHADOW_LDFLAGS += -lmklml
+ endif
+ MSHADOW_LDFLAGS += -liomp5 -L$(MKLROOT)/lib/
+endif
+
+ifeq ($(USE_BLAS), openblas)
+ MSHADOW_LDFLAGS += -lopenblas
+else ifeq ($(USE_BLAS), perfblas)
+ MSHADOW_LDFLAGS += -lperfblas
+else ifeq ($(USE_BLAS), atlas)
+ MSHADOW_LDFLAGS += $(ATLAS_LDFLAGS)
+else ifeq ($(USE_BLAS), blas)
+ MSHADOW_LDFLAGS += -lblas
+else ifeq ($(USE_BLAS), apple)
+ MSHADOW_CFLAGS += -I/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers/
+ MSHADOW_LDFLAGS += -framework Accelerate
+endif
+
+ifeq ($(PS_PATH), NONE)
+ PS_PATH = ..
+endif
+ifeq ($(PS_THIRD_PATH), NONE)
+ PS_THIRD_PATH = $(PS_PATH)/third_party
+endif
+
+ifndef RABIT_PATH
+ RABIT_PATH = rabit
+endif
+
+ifeq ($(RABIT_PATH), NONE)
+ RABIT_PATH = rabit
+endif
+
+ifeq ($(USE_RABIT_PS),1)
+ MSHADOW_CFLAGS += -I$(RABIT_PATH)/include
+ MSHADOW_LDFLAGS += -L$(RABIT_PATH)/lib -lrabit_base
+ MSHADOW_CFLAGS += -DMSHADOW_RABIT_PS=1
+else
+ MSHADOW_CFLAGS += -DMSHADOW_RABIT_PS=0
+endif
+
+ifeq ($(USE_DIST_PS),1)
+MSHADOW_CFLAGS += -DMSHADOW_DIST_PS=1 -std=c++11 \
+ -I$(PS_PATH)/src -I$(PS_THIRD_PATH)/include
+PS_LIB = $(addprefix $(PS_PATH)/build/, libps.a libps_main.a) \
+ $(addprefix $(PS_THIRD_PATH)/lib/, libgflags.a libzmq.a libprotobuf.a \
+ libglog.a libz.a libsnappy.a)
+ # -L$(PS_THIRD_PATH)/lib -lgflags -lzmq -lprotobuf -lglog -lz -lsnappy
+MSHADOW_NVCCFLAGS += --std=c++11
+else
+ MSHADOW_CFLAGS+= -DMSHADOW_DIST_PS=0
+endif
+
+# MSHADOW_USE_PASCAL=1 used to enable true-fp16 gemms. Now, mshadow
+# only uses pseudo-fp16 gemms, so this flag will be removed after
+# dependent projects no longer reference it.
+MSHADOW_CFLAGS += -DMSHADOW_USE_PASCAL=0
diff --git a/3rdparty/mshadow/mshadow-ps/.gitignore b/3rdparty/mshadow/mshadow-ps/.gitignore
new file mode 100644
index 0000000..076c1aa
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/.gitignore
@@ -0,0 +1,3 @@
+Makefile
+test
+test.cpp
diff --git a/3rdparty/mshadow/mshadow-ps/README.md b/3rdparty/mshadow/mshadow-ps/README.md
new file mode 100644
index 0000000..9c90cc9
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/README.md
@@ -0,0 +1,4 @@
+mshadow-ps
+====
+This folder contains mshadow-ps parameter server interface for mshadow GPU/CPU Tensor. See [guide on mshadow-ps](../guide/mshadow-ps) for introduction of the interface.
+
diff --git a/3rdparty/mshadow/mshadow-ps/mshadow_ps.h b/3rdparty/mshadow/mshadow-ps/mshadow_ps.h
new file mode 100644
index 0000000..a00e79a
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/mshadow_ps.h
@@ -0,0 +1,358 @@
+/*!
+ * Copyright (c) 2014 by Contributors
+ * \file mshadow_ps.h
+ * \brief parameter server abstraction for mshadow tensor
+ * this is a plugin of mshadow that can be used to syncrhonize
+ * parameters across device and machines
+ *
+ * \author Tianqi Chen, Mu Li
+ */
+#ifndef MSHADOW_PS_H_ // NOLINT(*)
+#define MSHADOW_PS_H_ // NOLINT(*)
+#include <vector>
+// optionally support of lambda function in C++11, if available
+#if __cplusplus >= 201103L
+#include <functional>
+#endif // C++11
+#include "../mshadow/tensor.h"
+
+/*! \brief whether to adapt distributed PS from parameter-server */
+#ifndef MSHADOW_DIST_PS
+#define MSHADOW_DIST_PS 1
+#endif
+
+/*! \brief whether to support BSP rabit API of PS*/
+#ifndef MSHADOW_RABIT_PS
+#define MSHADOW_RABIT_PS 1
+#endif
+
+namespace mshadow {
+/*! \brief namespace of mshadow-ps */
+namespace ps {
+/*!
+ * \brief interface of parameter server
+ * \tparam xpu the device of the data lies
+ * \tparam DType the type of element in the tensor
+ */
+template<typename xpu,
+ typename DType MSHADOW_DEFAULT_DTYPE>
+class ISharedModel {
+ public:
+ /*!
+ * \brief callback function that will be executed when pull request finishes
+ * before calling the callback, the thread context is already switched
+ * to the device of pullrequest
+ * \param stream the stream of callback thread, it is recommended to operate using this stream
+ * \param arg the argument of callback function
+ */
+ typedef void (CallbackFunction) (Stream<xpu> *stream, void *arg);
+ /*! \brief virtual destructor */
+ virtual ~ISharedModel(void) {}
+ /*!
+ * \brief Set param for the layer from string
+ * \param name parameter name
+ * \param val string for configuration
+ */
+ virtual void SetParam(const char *name, const char *val) {}
+ /*!
+ * \brief initialize the paramerver server client
+ * \param devices specifies the possible device id
+ * to be input from Push and Pull,
+ */
+ virtual void Init(const std::vector<int> &devices) {}
+ /*!
+ * \brief initialize the paramerver server client
+ * without specifying the devices, only device 0 is allowed
+ */
+ inline void Init(void) {
+ std::vector<int> dev;
+ dev.push_back(0);
+ this->Init(dev);
+ }
+ /*!
+ * \brief initialize a key with certain shape
+ * must be called before using Push/PullReq/PullWait
+ * on the corresponding key
+ * \param shape the shape content of the key
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ */
+ template<int dim>
+ inline void InitKey(Shape<dim> shape,
+ int key, int devid) {
+ this->InitKey_(shape.FlatTo2D(), key, devid);
+ }
+ /*!
+ * \brief wait until the pull event finishes
+ * if there was no pull request, wait will directly returns
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ */
+ virtual void PullWait(int key, int devid) = 0;
+ /*!
+ * \brief check if the weight was correct on the current device
+ *
+ * \param data the data
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ */
+ template<int dim>
+ inline void CheckWeight(Tensor<xpu, dim, DType> data,
+ int key,
+ int devid) {
+ this->CheckWeight_(data.FlatTo2D(), key, devid);
+ }
+ /*!
+ * \brief push out a tensor to parameter server
+ * this call is asynchronize and returns immediately
+ *
+ * \param data the data
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ * \param priority the priority of this operation,
+ * the bigger the number is the higher the priority will be
+ */
+ template<int dim>
+ inline void Push(Tensor<xpu, dim, DType> data,
+ int key,
+ int devid,
+ int priority = 0) {
+ this->Push_(data.FlatTo2D(), key, devid, priority);
+ }
+ /*!
+ * \brief send a pull request, to pull parameter into data
+ * this call is asynchronize and returns immediately
+ * use PullWait to wait the event of copy finish
+ *
+ * \param data the data
+ * \param key the unique key to indicate the tensor,
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ * \param priority the priority of this operation,
+ * the bigger the number is the higher the priority will be
+ * \param callback the callback function that will
+ * be invoked when the request finishes
+ * \param callback_arg the argument to pass to callback
+ */
+ template<int dim>
+ inline void PullReq(Tensor<xpu, dim, DType> data,
+ int key,
+ int devid,
+ int priority = 0,
+ CallbackFunction callback = NULL,
+ void *callback_arg = NULL) {
+ this->PullReq_(data.FlatTo2D(), key,
+ devid, priority, callback, callback_arg);
+ }
+#if __cplusplus >= 201103L
+ /*!
+ * \brief send a pull request, to pull parameter into data
+ * this call is asynchronize and returns immediately
+ * use PullWait to wait the event of copy finish
+ * this is the c++11 version that allows lambda function as callback
+ * \param data the data
+ * \param key the unique key to indicate the tensor,
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ * \param priority the priority of this operation,
+ * the bigger the number is the higher the priority will be
+ * \param callback the callback function
+ */
+ template<int dim>
+ inline void PullReq(Tensor<xpu, dim, DType> data,
+ int key,
+ int devid,
+ int priority,
+ std::function<void(Stream<xpu> *stream)> callback) {
+ // need to allocate space, because callback can happen latter..
+ auto calbk = new std::function<void(Stream<xpu> *stream)>();
+ *calbk = callback;
+ this->PullReq(data, key, devid, priority, InvokeLambda_, calbk);
+ }
+#endif // C++11
+
+ /*!
+ * \brief set weight of corresponding key in server
+ * this is a debug function that was not necessarily
+ * implemented by the server
+ * \param data the data to set
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ */
+ virtual void SetWeight_(Tensor<xpu, 2, DType> data,
+ int key,
+ int devid) = 0;
+ /*!
+ * \brief check if the weight matches the server side
+ * this is a debug function that was not necessarily
+ * implemented by the server
+ * \param data the data to set
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ */
+ virtual void CheckWeight_(Tensor<xpu, 2, DType> data,
+ int key,
+ int devid) = 0;
+
+ protected:
+ /*!
+ * \brief initialize a key with certain shape
+ * \param shape the shape content of the key
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ */
+ virtual void InitKey_(Shape<2> shape,
+ int key, int devid) = 0;
+ /*!
+ * \brief push out a tensor to parameter server
+ * this call is asynchronize and returns immediately
+ *
+ * \param data the data
+ * \param key the unique key to indicate the tensor
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ * \param priority the priority of this operation,
+ * the bigger the number is the higher the priority will be
+ */
+ virtual void Push_(Tensor<xpu, 2, DType> data,
+ int key,
+ int devid,
+ int priority = 0) = 0;
+ /*!
+ * \brief send a pull request, to pull parameter into data
+ * this call is asynchronize and returns immediately
+ * use PullWait to wait the event of copy finish
+ *
+ * \param data the data
+ * \param key the unique key to indicate the tensor,
+ * this is unique per device
+ * \param devid the device id this tensor lies in
+ * \param priority the priority of this operation,
+ * the bigger the number is the higher the priority will be
+ * \param callback the callback function that will
+ * be invoked when the request finishes
+ * \param callback_arg the argument to pass to callback
+ */
+ virtual void PullReq_(Tensor<xpu, 2, DType> data,
+ int key,
+ int devid,
+ int priority,
+ CallbackFunction callback,
+ void *callback_arg) = 0;
+
+ private:
+// C++11 support for lambda prepare function
+#if __cplusplus >= 201103L
+ /*! \brief hack function to convert lambda to callback function */
+ inline static void InvokeLambda_(Stream<xpu> *stream, void *fun) {
+ auto *fp = static_cast<std::function<void(Stream<xpu> *stream)>*>(fun);
+ (*fp)(stream);
+ delete fp;
+ }
+#endif // C++11
+};
+/*! \brief interface for customized mshadow server */
+template<typename DType>
+class IModelUpdater {
+ public:
+ virtual ~IModelUpdater(void) {}
+ /*!
+ * \brief set parameters from outside
+ * \param name name of parameter
+ * \param val value of parameter
+ */
+ virtual void SetParam(const char *name, const char *val) {}
+ /*!
+ * \brief init the model updater
+ * \param rank the rank of the node
+ * \param argc number of arguments
+ * \param argv arguments
+ */
+ virtual void InitUpdater(int rank, int argc, char *argv[]) {}
+ /*!
+ * \brief initialize the model
+ * \param key the key of data we point to
+ * \param dptr the data pointer
+ * \param size size of the parameter key
+ */
+ virtual void InitModel(int key, DType *dptr, size_t size) {
+ this->InitModel_(key, Tensor<cpu, 1, DType>(dptr, Shape1(size)));
+ }
+ /*!
+ * update the model
+ * \param key the key of data we point to
+ * \param dptr the data pointer
+ * \param size size of the parameter key
+ */
+ virtual void Update(int key, DType *dptr, size_t size) {
+ this->Update_(key, Tensor<cpu, 1, DType>(dptr, Shape1(size)));
+ }
+
+ protected:
+ /*!
+ * \brief initialize the model, user can implement this one
+ * to take advantage of tensor operations
+ * \param key the key of data we point to
+ * \param data the tensor data corresponding to the data we want to initialize
+ */
+ virtual void InitModel_(int key, Tensor<cpu, 1, DType> data) {
+ LOG(FATAL) << "InitModel: not implemented";
+ }
+ /*!
+ * \brief update the model, user can implement this one
+ * to take advantage of tensor operations
+ * \param key the key of data we point to
+ * \param data the tensor data corresponding to the data we want to initialize
+ */
+ virtual void Update_(int key, Tensor<cpu, 1, DType> data) {
+ LOG(FATAL) << "InitModel: not implemented";
+ }
+};
+/*!
+ * \brief create customized server
+ * this is a server defined by user
+ * \return new server
+ */
+template<typename DType>
+IModelUpdater<DType> *CreateModelUpdater(void);
+} // namespace ps
+} // namespace mshadow
+
+#include "./ps_local-inl.h"
+#include "./ps_dist-inl.h"
+#include "./ps_rabit-inl.h"
+namespace mshadow {
+namespace ps {
+/*!
+ * \brief create a parameter server implementation
+ * \param type the type of paramerver server
+ * can either be "local" or "dist"
+ * \return the ISharedModel that can be used to synchronize weights
+ */
+template<typename xpu, typename DType>
+inline ISharedModel<xpu, DType> *CreateSharedModel(const char *type) {
+ if (!strcmp("local", type)) {
+#if MSHADOW_RABIT_PS
+ // allreduce on one machine pays no cost
+ if (rabit::IsDistributed()) {
+ return new RabitModel<xpu, DType>();
+ }
+#endif
+ return new LocalModel<xpu, DType>();
+ }
+#if MSHADOW_DIST_PS
+ if (!strcmp("dist", type)) return new DistModel<xpu, DType>();
+#endif
+ LOG(FATAL) << "unknown server type " << type;
+ return NULL;
+}
+} // namespace ps
+} // namespace mshadow
+#endif // MSHADOW_PS_H_ NOLINT(*)
diff --git a/3rdparty/mshadow/mshadow-ps/ps_dist-inl.h b/3rdparty/mshadow/mshadow-ps/ps_dist-inl.h
new file mode 100644
index 0000000..9bce0e1
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/ps_dist-inl.h
@@ -0,0 +1,126 @@
+/*!
+ * Copyright (c) 2014 by Contributors
+ * \file ps_dist-inl.h
+ * \brief distributed version of PS
+ *
+ * \author Tianqi Chen, Mu Li
+ */
+#ifndef MSHADOW_PS_DIST_INL_H_ // NOLINT(*)
+#define MSHADOW_PS_DIST_INL_H_ // NOLINT(*)
+
+#include <vector>
+#include "./mshadow_ps.h"
+#include "./ps_local-inl.h"
+
+#if MSHADOW_DIST_PS
+#include "parameter/kv_layer.h"
+namespace mshadow {
+namespace ps {
+
+/**
+ * @brief bridge IModelUpdater to KVLayerUpdater
+ */
+template<typename DType>
+class UpdaterWrapper {
+ public:
+ explicit UpdaterWrapper(IModelUpdater<DType> * updater)
+ : updater_(updater) { }
+ ~UpdaterWrapper() { delete updater_; }
+
+ /// @brief initialize the data
+ void Init(int id, size_t size, DType* data) {
+ updater_->InitModel(id, data, size);
+ }
+
+ /// @brief update the model by using received data
+ void Update(int id, size_t size, const DType* recv_data, DType* data) {
+ updater_->Update(id, (DType*)recv_data, size); // NOLINT(*)
+ }
+ private:
+ IModelUpdater<DType> *updater_;
+};
+
+
+template<typename xpu, typename DType>
+class DistModel : public LocalModel<xpu, DType> {
+ public:
+ // parent type
+ typedef LocalModel<xpu, DType> Parent;
+
+ // initialize the parameter server
+ virtual void Init(const std::vector<int> &devices) {
+ Parent::Init(devices);
+ if (this->custom_server != NULL) {
+ delete this->custom_server;
+ this->custom_server = NULL;
+ }
+ }
+ virtual ~DistModel(void) {
+ }
+
+ protected:
+ // do nothing
+ virtual void InitCustomerServer(void) {
+ }
+ virtual void ServerInitKey(Tensor<cpu, 2> weight, int key) {
+ // this is called when key get initialized for the first time
+ // weight can be used to hold the model that pulled back
+ // use this to initialize the key on serverside
+ shared_model_.Pull(
+ ::ps::Parameter::Request(key), weight.dptr_, weight.MSize(),
+ [this, weight, key]() {
+ // call PullReady to notify LocalServer pulling is ready
+ this->PullReady(weight, key);
+ });
+ }
+ // override this function, to use parameter server
+ virtual void HandlePushFinish(Tensor<cpu, 3, DType> data,
+ int key) {
+ // summation the data fron all devices
+ LocalModel<xpu, DType>::ReduceSum(data);
+
+ // push and pull
+ Tensor<cpu, 2> sendrecv = data[0];
+ CHECK_EQ(data[0].CheckContiguous(), true) << "data must be contiguous";
+
+ int ts = shared_model_.Push(
+ ::ps::Parameter::Request(key), sendrecv.dptr_, sendrecv.MSize(), false);
+
+ // let this pull request wait the push finish at the server node
+ shared_model_.Pull(
+ ::ps::Parameter::Request(key, -1, {ts}), sendrecv.dptr_, sendrecv.MSize(),
+ [this, sendrecv, key]() {
+ // call PullReady to notify LocalServer pulling is ready
+ this->PullReady(sendrecv, key);
+ });
+ }
+
+ private:
+ ::ps::KVLayer<DType, UpdaterWrapper<DType> > shared_model_;
+};
+
+
+template<typename DType>
+class MShadowServerNode {
+ public:
+ // conf: get from the flag -app_conf
+ MShadowServerNode(int argc, char *argv[]) {
+ IModelUpdater<DType> *updater = CreateModelUpdater<DType>();
+ updater->InitUpdater(::ps::MyRank(), argc, argv);
+
+ UpdaterWrapper<DType> *wrapper = new UpdaterWrapper<DType>(updater);
+ typedef ::ps::KVLayer<DType, UpdaterWrapper<DType> > PSServer;
+ PSServer *shared_model_ = new PSServer();
+ shared_model_->set_updater(wrapper);
+ ::ps::Postoffice::instance().manager().TransferCustomer(
+ CHECK_NOTNULL(shared_model_));
+ }
+ virtual ~MShadowServerNode() { }
+};
+
+// NOTE: do not add PS::CreateServer here add it in the program that uses
+// mshadow-ps
+} // namespace ps
+} // namespace mshadow
+#endif
+#endif // MSHADOW_PS_DIST_INL_H_ NOLINT(*)
diff --git a/3rdparty/mshadow/mshadow-ps/ps_local-inl.h b/3rdparty/mshadow/mshadow-ps/ps_local-inl.h
new file mode 100644
index 0000000..ee4c55d
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/ps_local-inl.h
@@ -0,0 +1,814 @@
+/*!
+ * Copyright (c) 2014 by Contributors
+ * \file ps_local-inl.h
+ * \brief local multi-threading implementation of PS abstraction
+ *
+ * \author Tianqi Chen, Mu Li
+ */
+#ifndef MSHADOW_PS_LOCAL_INL_H_ // NOLINT(*)
+#define MSHADOW_PS_LOCAL_INL_H_ // NOLINT(*)
+#include <map>
+#include <utility>
+#include <string>
+#include <vector>
+#if defined(_OPENMP)
+#include <omp.h>
+#ifdef _MSC_VER
+typedef int ms_omp_uint;
+#else
+typedef unsigned ms_omp_uint;
+#endif
+#endif
+
+#include "./thread.h"
+#include "./thread_util.h"
+
+namespace mshadow {
+namespace ps {
+// multi-threaded implementation of
+template<typename xpu, typename DType>
+class LocalModel : public ISharedModel<xpu, DType> {
+ public:
+ // redefine callback function
+ typedef typename ISharedModel<xpu, DType>::CallbackFunction
+ CallbackFunction;
+ // constructor
+ LocalModel(void) {
+ init_end = 0;
+ perdev_pull_thread = 1;
+ perdev_push_thread = 1;
+ use_fifo_push_queue = 0;
+ bigarray_bound = 1000 * 1000;
+ nthread_reduction = 8;
+ use_pin_memory = 1;
+ test_on_server = 0;
+ update_on_server = 0;
+ destroy_signal = false;
+ custom_server = NULL;
+ }
+ // destructor
+ virtual ~LocalModel(void) {
+ this->Destroy();
+ }
+ inline void Destroy(void) {
+ if (init_end != 0) {
+ destroy_signal = true;
+ for (size_t i = 0; i < push_queues.size(); ++i) {
+ push_queues[i].Abort(1);
+ }
+ for (size_t i = 0; i < pull_queues.size(); ++i) {
+ pull_queues[i].Abort(1);
+ }
+ for (size_t i = 0; i < thread_push_handler.size(); ++i) {
+ thread_push_handler[i].Join();
+ }
+ for (size_t i = 0; i < thread_pull_handler.size(); ++i) {
+ thread_pull_handler[i].Join();
+ }
+ for (size_t i = 0; i < push_queues.size(); ++i) {
+ push_queues[i].Destroy();
+ }
+ push_map.Destroy();
+ push_lock.Destroy();
+ for (size_t i = 0; i < pull_queues.size(); ++i) {
+ pull_queues[i].Destroy();
+ }
+ pull_map.Destroy();
+ request_lock.Destroy();
+ wait_lock.Destroy();
+ wait_cond.Destroy();
+ init_end = 0;
+ }
+ if (custom_server != NULL) {
+ delete custom_server;
+ custom_server = NULL;
+ }
+ }
+ virtual void SetParam(const char *name, const char *val) {
+ int key;
+ if (sscanf(name, "push_op[%d]", &key) == 1) {
+ if (!strcmp(val, "gather")) {
+ request_lock.Lock();
+ push_operation[key] = kGather;
+ request_lock.Unlock();
+ return;
+ }
+ if (!strcmp(val, "sum")) {
+ push_operation[key] = kSum; return;
+ }
+ LOG(FATAL) << "unknown push operation " << val;
+ }
+ if (!strcmp(name, "reduce_thread")) {
+ nthread_reduction = atoi(val);
+ }
+ if (!strcmp(name, "use_pin_memory")) {
+ use_pin_memory = atoi(val);
+ }
+ if (!strcmp(name, "bigarray_bound")) {
+ bigarray_bound = static_cast<size_t>(atol(val));
+ }
+ if (!strcmp(name, "pull_thread")) {
+ if (!strcmp(val, "ndev")) {
+ perdev_pull_thread = 1;
+ } else if (!strcmp(val, "one")) {
+ perdev_pull_thread = 0;
+ } else {
+ LOG(FATAL) << "invalid value for parameter pull_thread," << " can only be ndev or one";
+ }
+ }
+ if (!strcmp(name, "push_thread")) {
+ if (!strcmp(val, "ndev")) {
+ perdev_push_thread = 1;
+ } else if (!strcmp(val, "one")) {
+ perdev_push_thread = 0;
+ } else {
+ LOG(FATAL) << "invalid value for parameter push_thread," << " can only be ndev or one";
+ }
+ }
+ if (!strcmp(name, "update_on_server")) {
+ update_on_server = atoi(val);
+ }
+ if (!strcmp(name, "test_on_server")) {
+ test_on_server = atoi(val);
+ }
+ // ignore message parameter
+ if (!strncmp(name, "msg:", 4)) return;
+ cfgvec.push_back(std::make_pair(std::string(name),
+ std::string(val)));
+ }
+ virtual void PullWait(int key, int devid) {
+ const int wid = GetWorkIndex(devid);
+ PullEntry *p = pull_map.Get(key);
+ if (p == NULL || p->wait.size() == 0) return;
+ PullEntry &e = *p;
+ // wake up waiters if any
+ CHECK_EQ(e.wait.size(), devices.size()) << "PullWait: must initialize the wait";
+ PullWaitRecord &w = e.wait[wid];
+ if (!w.finished) {
+ wait_lock.Lock();
+ w.nwait += 1;
+ while (!w.finished) {
+ wait_cond.Wait(&wait_lock);
+ }
+ w.nwait -= 1;
+ CHECK_GE(w.nwait, 0) << "boundary check";
+ wait_lock.Unlock();
+ }
+ }
+ virtual void Init(const std::vector<int> &devices) {
+ CHECK_EQ(init_end, 0) << "LocalServer.Init can only call Init once";
+ CHECK_NE(devices.size(), 0) << "LocalServer.Init: must at least contain 1 devices";
+ this->devices = devices;
+ destroy_signal = false;
+ // initialize device id to local index
+ dev2index.clear();
+ for (size_t i = 0; i < devices.size(); ++i) {
+ int devid = devices[i];
+ CHECK_GE(devid, 0) << "device id must be bigger than 0";
+ if (devid >= static_cast<int>(dev2index.size())) {
+ dev2index.resize(devid + 1, -1);
+ }
+ dev2index[devid] = static_cast<int>(i);
+ }
+ // allocate space
+ pull_stream.resize(devices.size());
+ push_stream.resize(devices.size());
+ // initialize all the thread related things
+ if (perdev_push_thread != 0) {
+ push_queues.resize(devices.size());
+ } else {
+ push_queues.resize(1);
+ }
+ for (size_t i = 0; i < push_queues.size(); ++i) {
+ push_queues[i].Init(use_fifo_push_queue != 0);
+ }
+ push_map.Init();
+ push_lock.Init();
+ pull_map.Init();
+ request_lock.Init();
+ wait_lock.Init();
+ wait_cond.Init();
+ if (perdev_pull_thread != 0) {
+ pull_queues.resize(devices.size());
+ } else {
+ pull_queues.resize(1);
+ }
+ for (size_t i = 0; i < pull_queues.size(); ++i) {
+ pull_queues[i].Init();
+ }
+ // initialize the thread
+ if (perdev_push_thread != 0) {
+ thread_push_handler.resize(devices.size());
+ for (size_t i = 0; i < devices.size(); ++i) {
+ std::pair<LocalModel*, size_t> *p
+ = new std::pair<LocalModel*, size_t>();
+ *p = std::make_pair(this, i);
+ thread_push_handler[i].Start(PushLocalThread, p);
+ }
+ } else {
+ thread_push_handler.resize(1);
+ thread_push_handler[0].Start(PushGlobalThread, this);
+ }
+ // initialize pull handler
+ if (perdev_pull_thread != 0) {
+ thread_pull_handler.resize(devices.size());
+ for (size_t i = 0; i < devices.size(); ++i) {
+ std::pair<LocalModel*, size_t> *p
+ = new std::pair<LocalModel*, size_t>();
+ *p = std::make_pair(this, i);
+ thread_pull_handler[i].Start(PullLocalThread, p);
+ }
+ } else {
+ thread_pull_handler.resize(1);
+ thread_pull_handler[0].Start(PullGlobalThread, this);
+ }
+ this->InitCustomerServer();
+ this->init_end = 1;
+ }
+
+ // set weight
+ virtual void SetWeight_(Tensor<xpu, 2, DType> data,
+ int key,
+ int devid) {
+ PushEntry &e = push_map.GetRef(key);
+ Stream<xpu> s;
+ push_lock.Lock();
+ mshadow::Copy(e.weight, data, &s);
+ push_lock.Unlock();
+ }
+ virtual void CheckWeight_(Tensor<xpu, 2, DType> data,
+ int key,
+ int devid) {
+ CHECK_NE(test_on_server, 0) << "must be in pair debug mode";
+ PushEntry &e = push_map.GetRef(key);
+ mshadow::TensorContainer<cpu, 2, DType> tmp(false);
+ tmp.Resize(data.shape_);
+ Stream<xpu> s;
+ push_lock.Lock();
+ // copy data
+ mshadow::Copy(tmp, data, &s);
+ index_t count = tmp.shape_.Size();
+ double diff = 0.0, ssum = 0.0, maxdiff = 0.0;
+ index_t mxidx = 0;
+ for (index_t i = 0; i < count; ++i) {
+ double d = std::abs(tmp.dptr_[i] - e.weight.dptr_[i]);
+ if (d > maxdiff) {
+ maxdiff = d; mxidx = i;
+ }
+ diff += d;
+ ssum += std::abs(tmp.dptr_[i]);
+ }
+ push_lock.Unlock();
+ // relative absolute error
+ double rerr = diff / ssum;
+ if (rerr > 1e-5 || diff != diff) {
+ fprintf(stderr, "PSLocal:key=%d,dev=%d: err=%f, maxd[%u]=%f, diff=%f, ssum=%f\n",
+ key, devid, rerr, mxidx, maxdiff, diff, ssum);
+ } else {
+ fprintf(stderr, "PSLocal:key=%d,dev=%d:check pass\n", key, devid);
+ }
+ }
+
+ protected:
+ /*! \brief operation performed locally in PS */
+ enum LocalOp {
+ /*! \brief take sum of all devices over the same key */
+ kSum = 0,
+ /*!
+ * \brief concatenate(gather),
+ * the tensors in all devices with same key
+ */
+ kGather = 1
+ };
+ virtual void InitKey_(Shape<2> shape,
+ int key, int devid) {
+ this->InitPullMap(key);
+ this->InitPushMap(key, shape);
+ }
+ virtual void Push_(Tensor<xpu, 2, DType> data,
+ int key, int devid, int priority) {
+ PullEntry &e = pull_map.GetRef(key);
+ e.req[GetWorkIndex(devid)].ready = false;
+ if (perdev_push_thread != 0) {
+ int wid = GetWorkIndex(devid);
+ push_queues[wid].Push(PullTask(data, key, devid), priority);
+ } else {
+ push_queues[0].Push(PullTask(data, key, devid), priority);
+ }
+ }
+ virtual void PullReq_(Tensor<xpu, 2, DType> data,
+ int key, int devid, int priority,
+ CallbackFunction callback,
+ void *callback_arg) {
+ PullEntry &e = pull_map.GetRef(key);
+ CHECK_EQ(e.req.size(), devices.size()) << "PullReq: must initialize the key, req";
+ CHECK_EQ(e.wait.size(), devices.size()) << "PullReq: must initialize the key, wait";
+ const int wid = GetWorkIndex(devid);
+ PullReqRecord &r = e.req[wid];
+ r.dest = data;
+ r.priority = priority;
+ r.callback = callback;
+ r.callback_arg = callback_arg;
+ // reset pull request finish mark
+ wait_lock.Lock();
+ e.wait[wid].finished = false;
+ wait_lock.Unlock();
+ // check ready event
+ request_lock.Lock();
+ CHECK_EQ(!r.pending, true) << "key = " << key
+ << "cannot send duplicate pull request before it finishes";
+ if (e.req[wid].ready) {
+ if (perdev_pull_thread != 0) {
+ pull_queues[wid].Push(std::make_pair(key, devid));
+ } else {
+ pull_queues[0].Push(std::make_pair(key, devid));
+ }
+ } else {
+ r.pending = true;
+ }
+ request_lock.Unlock();
+ }
+ /*!
+ * \brief called to notify that the data is ready for pull
+ * \param data the data that can be pulled back
+ * \param the key of the data
+ */
+ virtual void PullReady(Tensor<cpu, 2> data, int key) {
+ PullEntry &e = pull_map.GetRef(key);
+ CHECK_EQ(e.req.size(), devices.size()) << "PullReady: must initialize the key, req";
+ request_lock.Lock();
+ e.src = data;
+ for (index_t i = 0; i < e.req.size(); ++i) {
+ e.req[i].ready = true;
+ if (e.req[i].pending) {
+ if (perdev_pull_thread != 0) {
+ pull_queues[i].Push(std::make_pair(key, devices[i]));
+ } else {
+ pull_queues[0].Push(std::make_pair(key, devices[i]));
+ }
+ e.req[i].pending = false;
+ }
+ }
+ request_lock.Unlock();
+ }
+ virtual void ServerInitKey(Tensor<cpu, 2> weight, int key) {
+ if (custom_server != NULL) {
+ // intialize server, and ready for pullback
+ custom_server->InitModel(key, weight.dptr_, weight.MSize());
+ if (update_on_server != 0) {
+ this->PullReady(weight, key);
+ }
+ }
+ }
+ /*!
+ * \brief event handler for push finish
+ * called when all the data with same key comes int
+ * \param data the buffer holds the data in all devices
+ * \param key the key of the data
+ */
+ virtual void HandlePushFinish(Tensor<cpu, 3, DType> data,
+ int key) {
+ // LOG(ERROR) << dbstr(data);
+ LocalOp op = kSum;
+ typename std::map<int, LocalOp>::const_iterator
+ it = push_operation.find(key);
+ if (it != push_operation.end() && it->first == key) {
+ op = it->second;
+ }
+ // customized server
+ if (custom_server != NULL) {
+ this->ReduceSum(data);
+ custom_server->Update(key, data[0].dptr_, data[0].MSize());
+ if (update_on_server != 0) {
+ PushEntry &e = push_map.GetRef(key);
+ this->PullReady(e.weight, key);
+ } else {
+ CHECK_NE(test_on_server, 0) << "test mode";
+ this->PullReady(data[0], key);
+ }
+ return;
+ }
+ switch (op) {
+ case kSum: {
+ this->ReduceSum(data);
+ this->PullReady(data[0], key);
+ return;
+ }
+ case kGather: {
+ this->PullReady(data.FlatTo2D(), key);
+ return;
+ }
+ default: LOG(FATAL) << "unknown LocalOp";
+ }
+ }
+ /*!
+ * \brief event handler for reduce finish
+ * called when all the data with same key finishes the reduction
+ * \param data the buffer holds the reduction result
+ * \param key the key of the data
+ */
+ inline void HandleReduceFinish(Tensor<cpu, 2, DType> data,
+ int key) {
+ if (custom_server != NULL) {
+ custom_server->Update(key, data.dptr_, data.MSize());
+ if (update_on_server != 0) {
+ PushEntry &e = push_map.GetRef(key);
+ this->PullReady(e.weight, key);
+ } else {
+ CHECK_NE(test_on_server, 0) << "test mode";
+ this->PullReady(data, key);
+ }
+ } else {
+ this->PullReady(data, key);
+ }
+ }
+ virtual void InitCustomerServer(void) {
+ if (update_on_server != 0 || test_on_server != 0) {
+ custom_server = CreateModelUpdater<DType>();
+ for (size_t j = 0; j < cfgvec.size(); ++j) {
+ custom_server->SetParam(cfgvec[j].first.c_str(),
+ cfgvec[j].second.c_str());
+ }
+ custom_server->InitUpdater(0, 0, NULL);
+ }
+ }
+
+ protected:
+ // customized server
+ IModelUpdater<DType> *custom_server;
+ // whether use fifo push queue
+ int use_fifo_push_queue;
+
+ // perform sum reduction
+ inline void ReduceSum(Tensor<cpu, 3, DType> data) {
+ #if defined(_OPENMP)
+ if (data[0].MSize() >= bigarray_bound &&
+ nthread_reduction != 0) {
+ ms_omp_uint ntask = static_cast<ms_omp_uint>(data.size(1));
+ #pragma omp parallel for schedule(static) num_threads(nthread_reduction)
+ for (ms_omp_uint j = 0; j < ntask; ++j) {
+ for (index_t i = 1; i < data.size(0); ++i) {
+ data[0][j] += data[i][j];
+ }
+ }
+ } else //NOLINT(*)
+ #endif
+ {
+ for (index_t i = 1; i < data.size(0); ++i) {
+ data[0] += data[i];
+ }
+ }
+ }
+
+ private:
+ /*! \brief task running */
+ struct PullTask {
+ /*! \brief the task data source */
+ Tensor<xpu, 2, DType> data;
+ /*! \brief the key to the tensor */
+ int key;
+ /*!
+ * \brief the device id, (key,devid),
+ * uniquely identifies a mem location
+ */
+ int devid;
+ PullTask(void) {}
+ PullTask(Tensor<xpu, 2, DType> data, int key, int devid)
+ : data(data), key(key), devid(devid) {}
+ };
+ /*! \brief data structure to hold temporal push result */
+ struct PushEntry {
+ // temporal space to hold input data
+ Tensor<cpu, 4, DType> data;
+ // temporal space to hold weight, if needed
+ Tensor<cpu, 2, DType> weight;
+ // indicator whether the certain devices is already copied in
+ std::vector<bool> copied;
+ // number of data copied in
+ int num_copied;
+ // version number of data used to hold incomming data in push
+ int copyin_version;
+ // use pinned memory
+ bool pin_memory;
+ // constructor
+ PushEntry(void)
+ : copyin_version(0) {
+ weight.dptr_ = NULL;
+ }
+ ~PushEntry(void) {
+ if (data.dptr_ != NULL) {
+ if (pin_memory) {
+ mshadow::FreeHost<xpu>(&data);
+ if (weight.dptr_ != NULL) {
+ mshadow::FreeHost<xpu>(&weight);
+ }
+ } else {
+ mshadow::FreeSpace(&data);
+ if (weight.dptr_ != NULL) {
+ mshadow::FreeSpace(&weight);
+ }
+ }
+ }
+ }
+ // constructor
+ inline void Init(int ndevice, Shape<2> shape,
+ bool pin_memory, bool need_weight) {
+ this->pin_memory = pin_memory;
+ data.shape_ = Shape4(2, ndevice, shape[0], shape[1]);
+ weight.shape_ = shape;
+ if (pin_memory) {
+ mshadow::AllocHost<xpu>(&data);
+ if (need_weight) mshadow::AllocHost<xpu>(&weight);
+ } else {
+ mshadow::AllocSpace(&data, false);
+ if (need_weight) mshadow::AllocSpace(&weight);
+ }
+ CHECK_EQ(data.CheckContiguous(), true) << "Data must be contiguous";
+ CHECK(!need_weight || weight.CheckContiguous()) << "Weight must be contiguous";
+ num_copied = 0;
+ copied.resize(ndevice, false);
+ }
+ };
+ // a record to remember things related to pull request
+ struct PullReqRecord {
+ // whether this record contains a pending request
+ // whether pull is ready to go
+ bool ready;
+ // waiting for pull ready
+ bool pending;
+ // the destination to pull data into
+ Tensor<xpu, 2, DType> dest;
+ // the priority of the
+ int priority;
+ // callback function
+ CallbackFunction *callback;
+ // argument for callback
+ void *callback_arg;
+ PullReqRecord(void) : ready(false), pending(false) {
+ }
+ };
+ // a record to help handle pullwait
+ struct PullWaitRecord {
+ // number of thread that waits for the request to finish
+ int nwait;
+ // the request was finished
+ bool finished;
+ PullWaitRecord(void)
+ : nwait(0), finished(true) {
+ // set finished to true so pull without pull request returns
+ }
+ };
+ /*! \brief data structure to hold pull request */
+ struct PullEntry {
+ // data to be pulled back
+ Tensor<cpu, 2, DType> src;
+ // pullrequest record
+ std::vector<PullReqRecord> req;
+ // whether there is thread waiting on this event
+ std::vector<PullWaitRecord> wait;
+ PullEntry(void) {
+ }
+ };
+ // signal to notify all the thread about class destruction
+ bool destroy_signal;
+ // vector of devices
+ std::vector<int> devices;
+ // device index to local index
+ std::vector<int> dev2index;
+ //----- data structure used to support push ----
+ // stream used by push thread each device for memcpy
+ std::vector<Stream<xpu>*> push_stream;
+ // the queue used for push task
+ std::vector<utils::ThreadPQueue<PullTask> > push_queues;
+ // thread to handle push task
+ std::vector<utils::Thread> thread_push_handler;
+ // lock to lock push field
+ utils::Mutex push_lock;
+ // the map of push buffer
+ utils::ThreadSafeMap<PushEntry> push_map;
+ // customized local reduction operation
+ std::map<int, LocalOp> push_operation;
+ //----- data structure used to support pull ----
+ // the queue used for pull task
+ std::vector<utils::ThreadPQueue<std::pair<int, int> > > pull_queues;
+ // stream used by pull thread each device for memcpy
+ std::vector<Stream<xpu>*> pull_stream;
+ // the map to store pull status
+ utils::ThreadSafeMap<PullEntry> pull_map;
+ // thread to handle pull task
+ std::vector<utils::Thread> thread_pull_handler;
+ // lock to lock request field
+ utils::Mutex request_lock;
+ // lock to lock wait field
+ utils::Mutex wait_lock;
+ // conditional variable to do waiting
+ utils::ConditionVariable wait_cond;
+ // ---------configurations of server-------
+ int init_end;
+ // whether perform update on serverside
+ int update_on_server;
+ // debug option
+ int test_on_server;
+ // use pinned memory
+ int use_pin_memory;
+ // number of reduction thread
+ int nthread_reduction;
+ // the threshold for big array
+ size_t bigarray_bound;
+ // whether use pull thread per device
+ int perdev_pull_thread;
+ // whether use push thread per device
+ int perdev_push_thread;
+ /*! \brief history of configurations */
+ std::vector< std::pair<std::string, std::string> > cfgvec;
+ // push handler
+ inline void PushProc(utils::ThreadPQueue<PullTask> *queue) {
+ while (!destroy_signal) {
+ PullTask tsk;
+ if (queue->Pop(&tsk)) {
+ const int wid = GetWorkIndex(tsk.devid);
+ PushEntry &e = push_map.GetRef(tsk.key);
+ CHECK_EQ(e.data[0][0].shape_, tsk.data.shape_)
+ << "Tensor with same key must share same shape "
+ << e.data[0][0].shape_
+ << " vs "
+ << tsk.data.shape_;
+ CHECK_EQ(!e.copied[wid], true) << "data inconsistency";
+ // start copy
+ SetDevice<xpu>(tsk.devid);
+ Copy(e.data[e.copyin_version][wid], tsk.data, push_stream[wid]);
+ // wait till the copy finishes
+ push_stream[wid]->Wait();
+ // mark copied
+ e.copied[wid] = true;
+ push_lock.Lock();
+ e.num_copied += 1;
+ int cp_version = e.copyin_version;
+ bool push_finish = e.num_copied >= static_cast<int>(devices.size());
+ if (push_finish) {
+ // switch version
+ e.copyin_version = (e.copyin_version + 1) % e.data.size(0);
+ std::fill(e.copied.begin(), e.copied.end(), false);
+ e.num_copied = 0;
+ }
+ push_lock.Unlock();
+ if (push_finish) {
+ this->HandlePushFinish(e.data[cp_version], tsk.key);
+ }
+ } else {
+ CHECK_EQ(destroy_signal, true) << "abort but not destroy";
+ }
+ }
+ }
+ inline void PushHandlerGlobal(void) {
+ // allocate stream resources
+ for (size_t i = 0; i < devices.size(); ++i) {
+ SetDevice<xpu>(devices[i]);
+ push_stream[i] = NewStream<xpu>(devices[i]);
+ }
+ this->PushProc(&push_queues[0]);
+ // free resources
+ for (size_t i = 0; i < devices.size(); ++i) {
+ SetDevice<xpu>(devices[i]);
+ DeleteStream(push_stream[i]);
+ }
+ }
+ inline void PushHandlerLocal(size_t tid) {
+ CHECK_LT(tid, devices.size()) << "threadid exceed boundary";
+ CHECK_EQ(push_queues.size(), devices.size()) << "must have one pull_queue per device";
+ // allocate stream resources
+ SetDevice<xpu>(devices[tid]);
+ push_stream[tid] = NewStream<xpu>(devices[tid]);
+ this->PushProc(&push_queues[tid]);
+ SetDevice<xpu>(devices[tid]);
+ DeleteStream(push_stream[tid]);
+ }
+ /*!\brief entry point of loader thread */
+ inline static MSHADOW_THREAD_PREFIX PushGlobalThread(void *pthread) {
+ static_cast<LocalModel*>(pthread)->PushHandlerGlobal();
+ utils::ThreadExit(NULL);
+ return NULL;
+ }
+ inline static MSHADOW_THREAD_PREFIX PushLocalThread(void *arg) {
+ std::pair<LocalModel*, size_t> *p
+ = static_cast<std::pair<LocalModel*, size_t>*>(arg);
+ p->first->PushHandlerLocal(p->second);
+ delete p;
+ return NULL;
+ }
+ // push handler procedure
+ inline void PullProc(utils::ThreadPQueue<std::pair<int, int> > *queue) {
+ while (!destroy_signal) {
+ std::pair<int, int> tsk;
+ if (queue->Pop(&tsk)) {
+ const int key = tsk.first;
+ const int devid = tsk.second;
+ const int wid = GetWorkIndex(devid);
+ PullEntry &e = pull_map.GetRef(key);
+ {
+ // handle request
+ CHECK_EQ(e.req.size(), devices.size()) << "PullHandler: must initialize the key, req";
+ PullReqRecord &r = e.req[wid];
+ SetDevice<xpu>(devid);
+ Copy(r.dest, e.src, pull_stream[wid]);
+ // callback, if any
+ if (r.callback != NULL) {
+ (*r.callback)(pull_stream[wid], r.callback_arg);
+ }
+ // wait till the operation finishes
+ pull_stream[wid]->Wait();
+ }
+ {
+ // wake up waiters if any
+ CHECK_EQ(e.wait.size(), devices.size()) << "PullHandler, must initialize the key, req";
+ PullWaitRecord &w = e.wait[wid];
+ wait_lock.Lock();
+ w.finished = true;
+ if (w.nwait != 0) {
+ wait_cond.Broadcast();
+ }
+ wait_lock.Unlock();
+ }
+ } else {
+ CHECK_EQ(destroy_signal, true) << "abort but not destroy";
+ }
+ }
+ }
+ // use one thread for all pull actions
+ inline void PullHandlerGlobal(void) {
+ // allocate stream resources
+ for (size_t i = 0; i < devices.size(); ++i) {
+ SetDevice<xpu>(devices[i]);
+ pull_stream[i] = NewStream<xpu>(devices[i]);
+ }
+ this->PullProc(&pull_queues[0]);
+ // free resources
+ for (size_t i = 0; i < devices.size(); ++i) {
+ SetDevice<xpu>(devices[i]);
+ DeleteStream(pull_stream[i]);
+ }
+ }
+ inline void PullHandlerLocal(size_t tid) {
+ CHECK_LT(tid, devices.size()) << "threadid exceed boundary";
+ CHECK_EQ(pull_queues.size(), devices.size()) << "must have one pull_queue per device";
+ // allocate stream resources
+ SetDevice<xpu>(devices[tid]);
+ pull_stream[tid] = NewStream<xpu>(devices[tid]);
+ this->PullProc(&pull_queues[tid]);
+ SetDevice<xpu>(devices[tid]);
+ DeleteStream(pull_stream[tid]);
+ }
+ /*!\brief entry point of pull thread, one thread for all devices */
+ inline static MSHADOW_THREAD_PREFIX PullGlobalThread(void *arg) {
+ static_cast<LocalModel*>(arg)->PullHandlerGlobal();
+ return NULL;
+ }
+ inline static MSHADOW_THREAD_PREFIX PullLocalThread(void *arg) {
+ std::pair<LocalModel*, size_t> *p
+ = static_cast<std::pair<LocalModel*, size_t>*>(arg);
+ p->first->PullHandlerLocal(p->second);
+ delete p;
+ return NULL;
+ }
+ // get internal index of device
+ inline int GetWorkIndex(int devid) const {
+ CHECK(devid >= 0 &&
+ devid < static_cast<int>(dev2index.size()) &&
+ dev2index[devid] >= 0) << "Push: invalid devid";
+ return dev2index[devid];
+ }
+ // functions to handle pull
+ inline void InitPullMap(int key) {
+ pull_map.Init(key);
+ PullEntry &e = pull_map.GetRef(key);
+ request_lock.Lock();
+ // must recheck after lock
+ if (e.req.size() == 0) {
+ e.req.resize(devices.size(), PullReqRecord());
+ }
+ request_lock.Unlock();
+ // check wait map
+ wait_lock.Lock();
+ // must recheck after lock
+ if (e.wait.size() == 0) {
+ e.wait.resize(devices.size(), PullWaitRecord());
+ }
+ wait_lock.Unlock();
+ }
+ // functions to handle pull
+ inline void InitPushMap(int key, Shape<2> shape) {
+ push_map.Init(key);
+ PushEntry &e = push_map.GetRef(key);
+ push_lock.Lock();
+ if (e.copied.size() == 0) {
+ e.Init(devices.size(), shape,
+ use_pin_memory != 0,
+ update_on_server != 0 || test_on_server != 0);
+ }
+ this->ServerInitKey(e.weight, key);
+ push_lock.Unlock();
+ }
+};
+} // namespace ps
+} // namespace mshadow
+#endif // MSHADOW_PS_LOCAL_INL_H_ NOLINT(*)
diff --git a/3rdparty/mshadow/mshadow-ps/ps_rabit-inl.h b/3rdparty/mshadow/mshadow-ps/ps_rabit-inl.h
new file mode 100644
index 0000000..238932e
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/ps_rabit-inl.h
@@ -0,0 +1,113 @@
+/*!
+ * Copyright (c) 2014 by Contributors
+ * \file ps_rabit-inl.h
+ * \brief distributed version of PS using BSP
+ * synchronization in the backend
+ * \author Tianqi Chen, Mu Li
+ */
+#ifndef MSHADOW_PS_RABIT_INL_H_ // NOLINT(*)
+#define MSHADOW_PS_RABIT_INL_H_ // NOLINT(*)
+#include <vector>
+#include "./mshadow_ps.h"
+#include "./ps_local-inl.h"
+
+#if MSHADOW_RABIT_PS
+#include <rabit.h>
+namespace mshadow {
+namespace ps {
+// multi-threaded implementation of
+template<typename xpu, typename DType>
+class RabitModel : public LocalModel<xpu, DType> {
+ public:
+ // parent type
+ typedef LocalModel<xpu, DType> Parent;
+ // constructor
+ RabitModel() {
+ // enforce usage of fifo queue
+ this->use_fifo_push_queue = 1;
+ destroy_reduce_thread_ = false;
+ disable_allreduce_ = 0;
+ this->init_reducer_ = 0;
+ }
+ virtual ~RabitModel(void) {
+ Parent::Destroy();
+ if (init_reducer_ != 0) {
+ destroy_reduce_thread_ = true;
+ reduce_queue_.Abort(1);
+ thread_reduce_handler_.Join();
+ reduce_queue_.Destroy();
+ }
+ }
+ // initialize the parameter server
+ virtual void Init(const std::vector<int> &devices) {
+ this->use_fifo_push_queue = 1;
+ // use fifo
+ reduce_queue_.Init(true);
+ thread_reduce_handler_.Start(ReduceGlobalThread, this);
+ init_reducer_ = 1;
+ // initialize other things
+ Parent::Init(devices);
+ }
+ // set parameters
+ virtual void SetParam(const char *name, const char *val) {
+ if (!strcmp(name, "msg:disable_allreduce")) {
+ disable_allreduce_ = atoi(val);
+ }
+ Parent::SetParam(name, val);
+ }
+ // override this function, to use parameter server
+ virtual void HandlePushFinish(Tensor<cpu, 3, DType> data,
+ int key) {
+ // summation the data fron all devices
+ LocalModel<xpu, DType>::ReduceSum(data);
+ CHECK_EQ(data[0].CheckContiguous(), true) << "data must be contiguous";
+ ReduceTask tsk;
+ tsk.data = data[0]; tsk.key = key;
+ reduce_queue_.Push(tsk, 0);
+ }
+
+ private:
+ // reduce task
+ struct ReduceTask {
+ int key;
+ mshadow::Tensor<cpu, 2> data;
+ };
+ // destroy reduce
+ bool destroy_reduce_thread_;
+ // whether reducer is initialized
+ int init_reducer_;
+ // check disable_allreduce functionalities
+ int disable_allreduce_;
+ // reduce handler thread
+ utils::Thread thread_reduce_handler_;
+ // queue for allreduce task
+ utils::ThreadPQueue<ReduceTask> reduce_queue_;
+ // reduce handler
+ inline void ReduceHandler(void) {
+ while (!destroy_reduce_thread_) {
+ ReduceTask tsk;
+ if (reduce_queue_.Pop(&tsk)) {
+ CHECK_EQ(disable_allreduce_, 0) << "Allreduce disabled error";
+ int key = tsk.key;
+ rabit::Allreduce<rabit::op::Max>(&key, 1);
+ CHECK_EQ(key, tsk.key) << "Allreduce not concensus";
+ rabit::Allreduce<rabit::op::Sum>
+ (tsk.data.dptr_, tsk.data.MSize());
+ tsk.data *= 1.0f / rabit::GetWorldSize();
+ CHECK_EQ(disable_allreduce_, 0) << "Allreduce disabled error";
+ this->HandleReduceFinish(tsk.data, tsk.key);
+ } else {
+ CHECK_EQ(destroy_reduce_thread_, true) << "abort but not destroy";
+ }
+ }
+ }
+ /*!\brief entry point of reduce thread */
+ inline static MSHADOW_THREAD_PREFIX ReduceGlobalThread(void *pthread) {
+ static_cast<RabitModel*>(pthread)->ReduceHandler();
+ return NULL;
+ }
+};
+} // namespace ps
+} // namespace mshadow
+#endif // MSHADOW_RABIT_PS
+#endif // MSHADOW_PS_RABIT_INL_H_ // NOLINT(*)
diff --git a/3rdparty/mshadow/mshadow-ps/thread.h b/3rdparty/mshadow/mshadow-ps/thread.h
new file mode 100644
index 0000000..a80dffc
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/thread.h
@@ -0,0 +1,261 @@
+/*!
+ * Copyright by Contributors
+ * \file thread.h
+ * \brief this header include the minimum necessary resource
+ * for multi-threading that can be compiled in windows, linux, mac
+ * \author Tianqi Chen
+ */
+#ifndef MSHADOW_PS_THREAD_H_ // NOLINT(*)
+#define MSHADOW_PS_THREAD_H_ // NOLINT(*)
+
+#ifdef _MSC_VER
+#include <windows.h>
+#include <process.h>
+#include "../mshadow/logging.h"
+namespace mshadow {
+namespace utils {
+/*! \brief simple semaphore used for synchronization */
+class Semaphore {
+ public :
+ inline void Init(int init_val) {
+ sem = CreateSemaphore(NULL, init_val, 10, NULL);
+ CHECK_NE(sem, NULL) << "create Semaphore error";
+ }
+ inline void Destroy(void) {
+ CloseHandle(sem);
+ }
+ inline void Wait(void) {
+ CHECK_EQ(WaitForSingleObject(sem, INFINITE), WAIT_OBJECT_0)
+ << "WaitForSingleObject error";
+ }
+ inline void Post(void) {
+ CHECK_NE(ReleaseSemaphore(sem, 1, NULL), 0) << "ReleaseSemaphore error";
+ }
+
+ private:
+ HANDLE sem;
+};
+
+/*! \brief mutex under windows */
+class Mutex {
+ public:
+ inline void Init(void) {
+ CHECK_NE(InitializeCriticalSectionAndSpinCount(&mutex, 0x00000400), 0)
+ << "Mutex::Init fail";
+ }
+ inline void Lock(void) {
+ EnterCriticalSection(&mutex);
+ }
+ inline void Unlock(void) {
+ LeaveCriticalSection(&mutex);
+ }
+ inline void Destroy(void) {
+ DeleteCriticalSection(&mutex);
+ }
+
+ private:
+ friend class ConditionVariable;
+ CRITICAL_SECTION mutex;
+};
+
+// conditional variable that uses pthread
+class ConditionVariable {
+ public:
+ // initialize conditional variable
+ inline void Init(void) {
+ InitializeConditionVariable(&cond);
+ }
+ // destroy the thread
+ inline void Destroy(void) {
+ // DeleteConditionVariable(&cond);
+ }
+ // wait on the conditional variable
+ inline void Wait(Mutex *mutex) {
+ CHECK_NE(SleepConditionVariableCS(&cond, &(mutex->mutex), INFINITE), 0)
+ << "ConditionVariable:Wait fail";
+ }
+ inline void Broadcast(void) {
+ WakeAllConditionVariable(&cond);
+ }
+ inline void Signal(void) {
+ WakeConditionVariable(&cond);
+ }
+
+ private:
+ CONDITION_VARIABLE cond;
+};
+
+/*! \brief simple thread that wraps windows thread */
+class Thread {
+ private:
+ HANDLE thread_handle;
+ unsigned thread_id;
+ public:
+ inline void Start(unsigned int __stdcall entry(void*p), void *param) {
+ thread_handle = (HANDLE)_beginthreadex(NULL, 0, entry, param, 0, &thread_id);
+ }
+ inline int Join(void) {
+ WaitForSingleObject(thread_handle, INFINITE);
+ return 0;
+ }
+};
+/*! \brief exit function called from thread */
+inline void ThreadExit(void *status) {
+ _endthreadex(0);
+}
+#define MSHADOW_THREAD_PREFIX unsigned int __stdcall
+} // namespace utils
+} // namespace mshadow
+#else
+// thread interface using g++
+#include <semaphore.h>
+#include <pthread.h>
+#include <errno.h>
+namespace mshadow {
+namespace utils {
+/*!\brief semaphore class */
+class Semaphore {
+ #ifdef __APPLE__
+
+ private:
+ sem_t* semPtr;
+ char sema_name[20];
+
+ private:
+ inline void GenRandomString(char *s, const int len) {
+ static const char alphanum[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+ for (int i = 0; i < len; ++i) {
+ s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
+ }
+ s[len] = 0;
+ }
+
+ public:
+ inline void Init(int init_val) {
+ sema_name[0] = '/';
+ sema_name[1] = 's';
+ sema_name[2] = 'e';
+ sema_name[3] = '/';
+ GenRandomString(&sema_name[4], 16);
+ if ((semPtr = sem_open(sema_name, O_CREAT, 0644, init_val)) == SEM_FAILED) {
+ perror("sem_open");
+ exit(1);
+ }
+ CHECK_NE(semPtr, NULL) << "create Semaphore error";
+ }
+ inline void Destroy(void) {
+ if (sem_close(semPtr) == -1) {
+ perror("sem_close");
+ exit(EXIT_FAILURE);
+ }
+ if (sem_unlink(sema_name) == -1) {
+ perror("sem_unlink");
+ exit(EXIT_FAILURE);
+ }
+ }
+ inline void Wait(void) {
+ sem_wait(semPtr);
+ }
+ inline void Post(void) {
+ sem_post(semPtr);
+ }
+ #else
+
+ private:
+ sem_t sem;
+
+ public:
+ inline void Init(int init_val) {
+ if (sem_init(&sem, 0, init_val) != 0) {
+ LOG(FATAL) << "Semaphore.Init: " << strerror(errno);
+ }
+ }
+ inline void Destroy(void) {
+ if (sem_destroy(&sem) != 0) {
+ LOG(FATAL) << "Semaphore.Destroy: " << strerror(errno);
+ }
+ }
+ inline void Wait(void) {
+ if (sem_wait(&sem) != 0) {
+ LOG(FATAL) << "Semaphore.Wait: " << strerror(errno);
+ }
+ }
+ inline void Post(void) {
+ if (sem_post(&sem) != 0) {
+ LOG(FATAL) << "Semaphore.Post: " << strerror(errno);
+ }
+ }
+ #endif
+};
+
+// mutex that works with pthread
+class Mutex {
+ public:
+ inline void Init(void) {
+ pthread_mutex_init(&mutex, NULL);
+ }
+ inline void Lock(void) {
+ pthread_mutex_lock(&mutex);
+ }
+ inline void Unlock(void) {
+ pthread_mutex_unlock(&mutex);
+ }
+ inline void Destroy(void) {
+ pthread_mutex_destroy(&mutex);
+ }
+
+ private:
+ friend class ConditionVariable;
+ pthread_mutex_t mutex;
+};
+
+// conditional variable that uses pthread
+class ConditionVariable {
+ public:
+ // initialize conditional variable
+ inline void Init(void) {
+ pthread_cond_init(&cond, NULL);
+ }
+ // destroy the thread
+ inline void Destroy(void) {
+ pthread_cond_destroy(&cond);
+ }
+ // wait on the conditional variable
+ inline void Wait(Mutex *mutex) {
+ pthread_cond_wait(&cond, &(mutex->mutex));
+ }
+ inline void Broadcast(void) {
+ pthread_cond_broadcast(&cond);
+ }
+ inline void Signal(void) {
+ pthread_cond_signal(&cond);
+ }
+
+ private:
+ pthread_cond_t cond;
+};
+
+/*!\brief simple thread class */
+class Thread {
+ private:
+ pthread_t thread;
+ public :
+ inline void Start(void * entry(void*), void *param) { // NOLINT(*)
+ pthread_attr_t attr;
+ pthread_attr_init(&attr);
+ pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+ pthread_create(&thread, &attr, entry, param);
+ }
+ inline int Join(void) {
+ void *status;
+ return pthread_join(thread, &status);
+ }
+};
+inline void ThreadExit(void *status) {
+ pthread_exit(status);
+}
+} // namespace utils
+} // namespace mshadow
+#define MSHADOW_THREAD_PREFIX void *
+#endif // Linux
+#endif // MSHADOW_PS_THREAD_H_ NOLINT(*)
diff --git a/3rdparty/mshadow/mshadow-ps/thread_util.h b/3rdparty/mshadow/mshadow-ps/thread_util.h
new file mode 100644
index 0000000..185cfcf
--- /dev/null
+++ b/3rdparty/mshadow/mshadow-ps/thread_util.h
@@ -0,0 +1,169 @@
+/*!
+ * Copyright by Contributors
+ * \file thread_util.h
+ * \brief data structures for multi-threading communication
+ * \author Tianqi Chen
+ */
+#ifndef MSHADOW_PS_THREAD_UTIL_H_ // NOLINT(*)
+#define MSHADOW_PS_THREAD_UTIL_H_ // NOLINT(*)
+
+#include <utility>
+#include <queue>
+#include <map>
+#include "./thread.h"
+namespace mshadow {
+namespace utils {
+/*!
+ * \brief thread safe queue that can be used for customer consumer model
+ * in the future, it will support priority scheduling
+ * \tparam DType the content of the queue
+ */
+template<typename DType>
+class ThreadPQueue {
+ public:
+ // constructor
+ ThreadPQueue() : use_fifo_(false) {
+ }
+ /*! \brief intitialize the queue, must call this before use */
+ inline void Init(bool use_fifo = false) {
+ use_fifo_ = use_fifo;
+ lock_.Init();
+ counter_.Init(0);
+ }
+ /*! \brief destroy the resources on the queue */
+ inline void Destroy(void) {
+ lock_.Destroy();
+ counter_.Destroy();
+ }
+ /*!
+ * \brief Destroy the queue
+ * wake up all the threads waits on pop
+ * this is usually used in class destructor
+ * \param max_nthread the maximum number of thread that
+ * could be waiting on the queue
+ */
+ inline void Abort(int max_nthread = 1) {
+ for (int i = 0; i < max_nthread; ++i) {
+ counter_.Post();
+ }
+ }
+ /*!
+ * \brief push an element to the queue
+ * \param data the data to be puhed into queue
+ * \param optionally priority level to hint which
+ * element should be poped first
+ */
+ inline void Push(const DType &data, int priority = 0) {
+ lock_.Lock();
+ if (use_fifo_) {
+ fqueue_.push(data);
+ } else {
+ pqueue_.push(Entry(data, priority));
+ }
+ lock_.Unlock();
+ counter_.Post();
+ }
+ /*!
+ * \brief pop an element from the queue
+ * this will block the thread if the queue is empty
+ * \param data_out the address to put output of the queue
+ * \return true if a correct element is returned
+ * false if abort is called and no element was left in queue
+ */
+ inline bool Pop(DType *data_out) {
+ counter_.Wait();
+ lock_.Lock();
+ if (use_fifo_) {
+ if (fqueue_.size() == 0) {
+ lock_.Unlock(); return false;
+ }
+ } else {
+ if (pqueue_.size() == 0) {
+ lock_.Unlock(); return false;
+ }
+ }
+ if (use_fifo_) {
+ CHECK_NE(fqueue_.size(), 0) << "Queue.Pop";
+ *data_out = fqueue_.front();
+ fqueue_.pop();
+ } else {
+ CHECK_NE(pqueue_.size(), 0) << "Queue.Pop";
+ *data_out = pqueue_.top().data;
+ pqueue_.pop();
+ }
+ lock_.Unlock();
+ return true;
+ }
+
+ private:
+ // entry in the queue
+ struct Entry {
+ DType data;
+ int priority;
+ Entry(const DType &data, int priority)
+ : data(data), priority(priority) {}
+ inline bool operator<(const Entry &b) const {
+ return priority < b.priority;
+ }
+ };
+ // whether use FIFO queue
+ bool use_fifo_;
+ // a priority queue
+ std::priority_queue<Entry> pqueue_;
+ // a FIFO queue
+ std::queue<DType> fqueue_;
+ // lock for accessing the queue
+ utils::Mutex lock_;
+ // counter to count number of push tasks
+ utils::Semaphore counter_;
+};
+
+// naive implementation of threadsafe map
+template<typename TValue>
+class ThreadSafeMap {
+ public:
+ inline void Init(void) {
+ lock_.Init();
+ }
+ inline void Destroy(void) {
+ for (typename std::map<int, TValue*>::iterator
+ it = map_.begin(); it != map_.end(); ++it) {
+ delete it->second;
+ }
+ lock_.Destroy();
+ }
+ inline TValue *Get(int key) {
+ TValue *ret;
+ lock_.Lock();
+ typename std::map<int, TValue*>::const_iterator
+ it = map_.find(key);
+ if (it == map_.end() || it->first != key) {
+ ret = NULL;
+ } else {
+ ret = it->second;
+ }
+ lock_.Unlock();
+ return ret;
+ }
+ inline TValue &GetRef(int key) {
+ TValue *ret = this->Get(key);
+ CHECK_NE(ret, NULL) << "key = " << key << " does not exist";
+ return *ret;
+ }
+ inline void Init(int key) {
+ lock_.Lock();
+ if (map_.count(key) == 0) {
+ map_[key] = new TValue();
+ }
+ lock_.Unlock();
+ }
+
+ private:
+ // lock for accessing the queue
+ utils::Mutex lock_;
+ std::map<int, TValue*> map_;
+};
+
+} // namespace utils
+} // namespace mshadow
+#endif // MSHADOW_PS_THREAD_UTIL_H_
diff --git a/3rdparty/mshadow/mshadow/README.md b/3rdparty/mshadow/mshadow/README.md
new file mode 100644
index 0000000..86276af
--- /dev/null
+++ b/3rdparty/mshadow/mshadow/README.md
@@ -0,0 +1,8 @@
+Code Guide
+====
+This readme contains notes about code in mshadow. MShadow generally follows Google's C++ Style.
+
+Convention
+====
+* Basically, all the files ends in ```-inl.h, -inl.cuh``` are implementations, and can be ignored if only using mshadow
+* The files ends in ```.h``` are heavily commented with [doxyen format](http://www.doxygen.org/), and can be used to generate the corresponding document.
diff --git a/3rdparty/mshadow/mshadow/base.h b/3rdparty/mshadow/mshadow/base.h
new file mode 100755
index 0000000..d08efd3
--- /dev/null
+++ b/3rdparty/mshadow/mshadow/base.h
@@ -0,0 +1,1110 @@
+/*!
+ * Copyright (c) 2014 by Contributors
+ * \file base.h
+ * \brief definitions of base types, operators, macros functions
+ *
+ * \author Bing Xu, Tianqi Chen
+ */
+#ifndef MSHADOW_BASE_H_
+#define MSHADOW_BASE_H_
+#ifdef _MSC_VER
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+#ifndef _CRT_SECURE_NO_DEPRECATE
+#define _CRT_SECURE_NO_DEPRECATE
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#endif
+#include <cmath>
+#include <cstdio>
+#include <cfloat>
+#include <climits>
+#include <algorithm>
+#include <functional>
+#include <sstream>
+#include <string>
+
+#ifdef _MSC_VER
+//! \cond Doxygen_Suppress
+typedef signed char int8_t;
+typedef __int16 int16_t;
+typedef __int32 int32_t;
+typedef __int64 int64_t;
+typedef unsigned char uint8_t;
+typedef unsigned __int16 uint16_t;
+typedef unsigned __int32 uint32_t;
+typedef unsigned __int64 uint64_t;
+//! \endcond
+#else
+#include <inttypes.h>
+#endif
+// macro defintiions
+/*!
+ * \brief if this macro is define to be 1,
+ * mshadow should compile without any of other libs
+ */
+#ifndef MSHADOW_STAND_ALONE
+#define MSHADOW_STAND_ALONE 0
+#endif
+/*! \brief whether do padding during allocation */
+#ifndef MSHADOW_ALLOC_PAD
+#define MSHADOW_ALLOC_PAD true
+#endif
+/*!
+ * \brief
+ * x dimension of data must be bigger pad_size * ratio to be alloced padded memory,
+ * otherwise use tide allocation
+ * for example, if pad_ratio=2, GPU memory alignement size is 32,
+ * then we will only allocate padded memory if x dimension > 64
+ * set it to 0 then we will always allocate padded memory
+ */
+#ifndef MSHADOW_MIN_PAD_RATIO
+ #define MSHADOW_MIN_PAD_RATIO 2
+#endif
+
+#if MSHADOW_STAND_ALONE
+ #define MSHADOW_USE_CBLAS 0
+ #define MSHADOW_USE_MKL 0
+ #define MSHADOW_USE_CUDA 0
+#endif
+
+/*!
+ * \brief force user to use GPU stream during computation
+ * error will be shot when default stream NULL is used
+ */
+#ifndef MSHADOW_FORCE_STREAM
+#define MSHADOW_FORCE_STREAM 1
+#endif
+
+/*! \brief use CBLAS for CBLAS */
+#ifndef MSHADOW_USE_CBLAS
+ #define MSHADOW_USE_CBLAS 0
+#endif
+/*! \brief use MKL for BLAS */
+#ifndef MSHADOW_USE_MKL
+ #define MSHADOW_USE_MKL 1
+#endif
+
+/*!
+ * \brief use CUDA support, must ensure that the cuda include path is correct,
+ * or directly compile using nvcc
+ */
+#ifndef MSHADOW_USE_CUDA
+ #define MSHADOW_USE_CUDA 1
+#endif
+
+/*!
+ * \brief use CUDNN support, must ensure that the cudnn include path is correct
+ */
+#ifndef MSHADOW_USE_CUDNN
+ #define MSHADOW_USE_CUDNN 0
+#endif
+
+/*!
+ * \brief use CUSOLVER support
+ */
+#ifndef MSHADOW_USE_CUSOLVER
+ #define MSHADOW_USE_CUSOLVER MSHADOW_USE_CUDA
+#endif
+
+/*!
+ * \brief seems CUDAARCH is deprecated in future NVCC
+ * set this to 1 if you want to use CUDA version smaller than 2.0
+ */
+#ifndef MSHADOW_OLD_CUDA
+#define MSHADOW_OLD_CUDA 0
+#endif
+
+/*!
+ * \brief macro to decide existence of c++11 compiler
+ */
+#ifndef MSHADOW_IN_CXX11
+ #if (defined(__GXX_EXPERIMENTAL_CXX0X__) ||\
+ __cplusplus >= 201103L || defined(_MSC_VER))
+ #define MSHADOW_IN_CXX11 1
+ #else
+ #define MSHADOW_IN_CXX11 0
+ #endif
+#endif
+
+/*! \brief whether use SSE */
+#ifndef MSHADOW_USE_SSE
+ #define MSHADOW_USE_SSE 1
+#endif
+
+/*! \brief whether use F16C instruction set architecture extension */
+#ifndef MSHADOW_USE_F16C
+ #if defined(_MSC_VER) || defined(__CUDACC__)
+ #define MSHADOW_USE_F16C 0
+ #elif defined(__clang__) && \
+ ((__clang_major__ < 8) || ((__clang_major__ == 8) && (__clang_minor__ < 1)))
+ #define MSHADOW_USE_F16C 0
+ #else
+ #define MSHADOW_USE_F16C 1
+ #endif
+#endif
+
+/*! \brief whether use NVML to get dynamic info */
+#ifndef MSHADOW_USE_NVML
+ #define MSHADOW_USE_NVML 0
+#endif
+// SSE is conflict with cudacc
+#ifdef __CUDACC__
+ #undef MSHADOW_USE_SSE
+ #define MSHADOW_USE_SSE 0
+#endif
+
+#if MSHADOW_USE_CBLAS
+extern "C" {
+ #include <cblas.h>
+}
+#elif MSHADOW_USE_MKL
+ #include <mkl_blas.h>
+ #include <mkl_cblas.h>
+ #include <mkl_vsl.h>
+ #include <mkl_vsl_functions.h>
+ #include <mkl_version.h>
+#endif
+
+#if MSHADOW_USE_CUDA
+ #include <cuda.h>
+ #include <cublas_v2.h>
+ #include <curand.h>
+#endif
+
+#if MSHADOW_USE_CUDNN == 1
+ #include <cudnn.h>
+#endif
+
+#if MSHADOW_USE_CUSOLVER == 1
+ #include <cusolverDn.h>
+#endif
+
+#if MSHADOW_USE_NVML
+ #include <nvml.h>
+#endif
+
+// --------------------------------
+// MSHADOW_XINLINE is used for inlining template code for both CUDA and CPU code
+#ifdef MSHADOW_XINLINE
+ #error "MSHADOW_XINLINE must not be defined"
+#endif
+#ifdef _MSC_VER
+#define MSHADOW_FORCE_INLINE __forceinline
+#pragma warning(disable : 4068)
+#else
+#define MSHADOW_FORCE_INLINE inline __attribute__((always_inline))
+#endif
+#ifdef __CUDACC__
+ #define MSHADOW_XINLINE MSHADOW_FORCE_INLINE __device__ __host__
+#else
+ #define MSHADOW_XINLINE MSHADOW_FORCE_INLINE
+#endif
+/*! \brief cpu force inline */
+#define MSHADOW_CINLINE MSHADOW_FORCE_INLINE
+
+#if defined(__GXX_EXPERIMENTAL_CXX0X) ||\
+ defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L
+ #define MSHADOW_CONSTEXPR constexpr
+#else
+ #define MSHADOW_CONSTEXPR const
+#endif
+
+/*!
+ * \brief default data type for tensor string
+ * in code release, change it to default_real_t
+ * during development, change it to empty string so that missing
+ * template arguments can be detected
+ */
+#ifndef MSHADOW_DEFAULT_DTYPE
+#define MSHADOW_DEFAULT_DTYPE = ::mshadow::default_real_t
+#endif
+
+/*!
+ * \brief DMLC marco for logging
+ */
+#ifndef MSHADOW_USE_GLOG
+#define MSHADOW_USE_GLOG DMLC_USE_GLOG
+#endif // MSHADOW_USE_GLOG
+
+#if DMLC_USE_CXX11
+#define MSHADOW_THROW_EXCEPTION noexcept(false)
+#define MSHADOW_NO_EXCEPTION noexcept(true)
+#else
+#define MSHADOW_THROW_EXCEPTION
+#define MSHADOW_NO_EXCEPTION
+#endif
+
+#if defined(_MSC_VER)
+#define MSHADOW_ALIGNED(x) __declspec(align(x))
+#else
+#define MSHADOW_ALIGNED(x) __attribute__ ((aligned(x)))
+#endif
+
+/*!
+ * \brief Protected cuda call in mshadow
+ * \param func Expression to call.
+ * It checks for CUDA errors after invocation of the expression.
+ */
+#define MSHADOW_CUDA_CALL(func) \
+ { \
+ cudaError_t e = (func); \
+ if (e == cudaErrorCudartUnloading) { \
+ throw dmlc::Error(cudaGetErrorString(e)); \
+ } \
+ CHECK(e == cudaSuccess) \
+ << "CUDA: " << cudaGetErrorString(e); \
+ }
+
+/*!
+ * \brief Run function and catch error, log unknown error.
+ * \param func Expression to call.
+ */
+#define MSHADOW_CATCH_ERROR(func) \
+ { \
+ try { \
+ (func); \
+ } catch (const dmlc::Error &e) { \
+ std::string what = e.what(); \
+ if (what.find("driver shutting down") == std::string::npos) { \
+ LOG(ERROR) << "Ignore CUDA Error " << what; \
+ } \
+ } \
+ }
+
+#include "./half.h"
+#include "./half2.h"
+#include "./logging.h"
+/*! \brief namespace for mshadow */
+namespace mshadow {
+/*! \brief buffer size for each random number generator */
+const unsigned kRandBufferSize = 1000000;
+/*! \brief pi */
+const float kPi = 3.1415926f;
+/*! \brief type that will be used for index */
+#if MSHADOW_INT64_TENSOR_SIZE == 1
+ typedef int64_t index_t;
+#else
+ typedef int32_t index_t;
+#endif
+
+#ifdef _WIN32
+ /*! \brief openmp index for windows */
+ typedef int64_t openmp_index_t;
+#else
+ /*! \brief openmp index for linux */
+ typedef index_t openmp_index_t;
+#endif
+
+/*! \brief float point type that will be used in default by mshadow */
+typedef float default_real_t;
+
+/*! \brief data type flag */
+enum TypeFlag {
+ kFloat32 = 0,
+ kFloat64 = 1,
+ kFloat16 = 2,
+ kUint8 = 3,
+ kInt32 = 4,
+ kInt8 = 5,
+ kInt64 = 6,
+};
+
+template<typename DType>
+struct DataType;
+
+template<>
+struct DataType<float> {
+ static const int kFlag = kFloat32;
+ static const int kLanes = 1;
+#if MSHADOW_USE_CUDA
+#if (CUDA_VERSION >= 8000)
+ static const cudaDataType_t kCudaFlag = CUDA_R_32F;
+#endif
+#if MSHADOW_USE_CUDNN
+ static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_FLOAT;
+ typedef float ScaleType;
+#endif
+#endif
+};
+template<>
+struct DataType<double> {
+ static const int kFlag = kFloat64;
+ static const int kLanes = 1;
+#if MSHADOW_USE_CUDA
+#if (CUDA_VERSION >= 8000)
+ static const cudaDataType_t kCudaFlag = CUDA_R_64F;
+#endif
+#if MSHADOW_USE_CUDNN
+ static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_DOUBLE;
+ typedef double ScaleType;
+#endif
+#endif
+};
+template<>
+struct DataType<half::half_t> {
+ static const int kFlag = kFloat16;
+ static const int kLanes = 1;
+#if MSHADOW_USE_CUDA
+#if (CUDA_VERSION >= 8000)
+ static const cudaDataType_t kCudaFlag = CUDA_R_16F;
+#endif
+#if MSHADOW_USE_CUDNN
+ static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_HALF;
+ typedef float ScaleType;
+#endif
+#endif
+};
+template<>
+struct DataType<half::half2_t> {
+ static const int kFlag = kFloat16;
+ static const int kLanes = 2;
+};
+template<>
+struct DataType<uint8_t> {
+ static const int kFlag = kUint8;
+ static const int kLanes = 1;
+#if MSHADOW_USE_CUDA
+#if (CUDA_VERSION >= 8000)
+ static const cudaDataType_t kCudaFlag = CUDA_R_8U;
+#endif
+#if (MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 6)
+ // no uint8 in cudnn for now
+ static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_INT8;
+ typedef uint8_t ScaleType;
+#endif
+#endif
+};
+template<>
+struct DataType<int8_t> {
+ static const int kFlag = kInt8;
+ static const int kLanes = 1;
+#if MSHADOW_USE_CUDA
+#if (CUDA_VERSION >= 8000)
+ static const cudaDataType_t kCudaFlag = CUDA_R_8I;
+#endif
+#if (MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 6)
+ static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_INT8;
+ typedef int8_t ScaleType;
+#endif
+#endif
+};
+template<>
+struct DataType<int32_t> {
+ static const int kFlag = kInt32;
+ static const int kLanes = 1;
+#if MSHADOW_USE_CUDA
+#if (CUDA_VERSION >= 8000)
+ static const cudaDataType_t kCudaFlag = CUDA_R_32I;
+#endif
+#if (MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 6)
+ static const cudnnDataType_t kCudnnFlag = CUDNN_DATA_INT32;
+ typedef int32_t ScaleType;
+#endif
+#endif
+};
+template<>
+struct DataType<int64_t> {
+ static const int kFlag = kInt64;
+ static const int kLanes = 1;
+};
+
+/*! \brief type enum value for default real type */
+const int default_type_flag = DataType<default_real_t>::kFlag;
+
+/*! layout flag */
+enum LayoutFlag {
+ kNCHW = 0,
+ kNHWC,
+ kCHWN,
+
+ kNCW = 1 << 3,
+ kNWC,
+ kCWN,
+
+ kNCDHW = 1 << 5,
+ kNDHWC,
+ kCDHWN
+};
+
+template<int layout>
+struct LayoutType;
+
+template<>
+struct LayoutType<kNCHW> {
+ static const index_t kNdim = 4;
+#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)
+ static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NCHW;
+#else
+ static const int kCudnnFlag = -1;
+#endif
+};
+
+template<>
+struct LayoutType<kNHWC> {
+ static const index_t kNdim = 4;
+#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)
+ static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NHWC;
+#else
+ static const int kCudnnFlag = -1;
+#endif
+};
+
+/*! \brief default layout for 4d tensor */
+const int default_layout = kNCHW;
+
+template<>
+struct LayoutType<kNCDHW> {
+ static const index_t kNdim = 5;
+#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)
+ static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NCHW;
+#else
+ static const int kCudnnFlag = -1;
+#endif
+};
+
+template<>
+struct LayoutType<kNDHWC> {
+ static const index_t kNdim = 5;
+#if (MSHADOW_USE_CUDA && MSHADOW_USE_CUDNN == 1 && CUDNN_MAJOR >= 4)
+ static const cudnnTensorFormat_t kCudnnFlag = CUDNN_TENSOR_NHWC;
+#else
+ static const int kCudnnFlag = -1;
+#endif
+};
+
+/*! \brief default layout for 5d tensor */
+const int default_layout_5d = kNCDHW;
+
+/*! \brief namespace for operators */
+namespace op {
+// binary operator
+/*! \brief mul operator */
+struct mul{
+ /*! \brief map a, b to result using defined operation */
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a, DType b) {
+ return a * b;
+ }
+};
+/*! \brief plus operator */
+struct plus {
+ /*! \brief map a, b to result using defined operation */
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a, DType b) {
+ return a + b;
+ }
+};
+/*! \brief minus operator */
+struct minus {
+ /*! \brief map a, b to result using defined operation */
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a, DType b) {
+ return a - b;
+ }
+};
+/*! \brief divide operator */
+struct div {
+ /*! \brief map a, b to result using defined operation */
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a, DType b) {
+ return a / b;
+ }
+};
+/*! \brief get rhs */
+struct right {
+ /*! \brief map a, b to result using defined operation */
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a, DType b) {
+ return b;
+ }
+};
+// unary operator/ function: example
+// these operators can be defined by user,
+// in the same style as binary and unary operator
+// to use, simply write F<op::identity>( src )
+/*! \brief identity function that maps a real number to it self */
+struct identity{
+ /*! \brief map a to result using defined operation */
+ template<typename DType>
+ MSHADOW_XINLINE static DType Map(DType a) {
+ return a;
+ }
+};
+} // namespace op
+/*! \brief namespace for savers */
+namespace sv {
+/*! \brief save to saver: = */
+struct saveto {
+ /*! \brief save b to a using save method */
+ template<typename DType>
+ MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)
+ a = b;
+ }
+ /*! \brief helper constant to use BLAS, alpha */
+ inline static default_real_t AlphaBLAS(void) { return 1.0f; }
+ /*! \brief helper constant to use BLAS, beta */
+ inline static default_real_t BetaBLAS(void) { return 0.0f; }
+ /*! \brief corresponding binary operator type */
+ typedef op::right OPType;
+};
+/*! \brief save to saver: += */
+struct plusto {
+ /*! \brief save b to a using save method */
+ template<typename DType>
+ MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)
+ a += b;
+ }
+ /*! \brief helper constant to use BLAS, alpha */
+ inline static default_real_t AlphaBLAS(void) { return 1.0f; }
+ /*! \brief helper constant to use BLAS, beta */
+ inline static default_real_t BetaBLAS(void) { return 1.0f; }
+ /*! \brief corresponding binary operator type */
+ typedef op::plus OPType;
+};
+/*! \brief minus to saver: -= */
+struct minusto {
+ /*! \brief save b to a using save method */
+ template<typename DType>
+ MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)
+ a -= b;
+ }
+ /*! \brief helper constant to use BLAS, alpha */
+ inline static default_real_t AlphaBLAS(void) { return -1.0f; }
+ /*! \brief helper constant to use BLAS, beta */
+ inline static default_real_t BetaBLAS(void) { return 1.0f; }
+ /*! \brief corresponding binary operator type */
+ typedef op::minus OPType;
+};
+/*! \brief multiply to saver: *= */
+struct multo {
+ /*! \brief save b to a using save method */
+ template<typename DType>
+ MSHADOW_XINLINE static void Save(DType &a, DType b) { // NOLINT(*)
+ a *= b;
+ }
+ /*! \brief corresponding binary operator type */
+ typedef op::mul OPType;
+};
+/*! \brief divide to saver: /= */
+struct divto {
+ /*! \brief save b to a using save method */
+ template<typename DType>
+ MSHADOW_XINLINE static void Save(DType& a, DType b) { // NOLINT(*)
+ a /= b;
+ }
+ /*! \brief corresponding binary operator type */
+ typedef op::div OPType;
+};
+} // namespace sv
+/*! \brief namespace for potential reducer operations */
+namespace red {
+namespace limits {
+/*!
+ * \brief minimum value of certain types
+ * \tparam DType data type
+ */
+template<typename DType>
+MSHADOW_XINLINE DType MinValue(void);
+/*! \brief minimum value of float */
+template<>
+MSHADOW_XINLINE float MinValue<float>(void) {
+ return -FLT_MAX;
+}
+/*! \brief minimum value of double */
+template<>
+MSHADOW_XINLINE double MinValue<double>(void) {
+ return -DBL_MAX;
+}
+/*! \brief minimum value of half */
+template<>
+MSHADOW_XINLINE half::half_t MinValue<half::half_t>(void) {
+ return MSHADOW_HALF_MIN;
+}
+/*! \brief minimum value of uint8_t */
+template<>
+MSHADOW_XINLINE uint8_t MinValue<uint8_t>(void) {
+ return 0;
+}
+/*! \brief minimum value of int8_t */
+template<>
+MSHADOW_XINLINE int8_t MinValue<int8_t>(void) {
+ return SCHAR_MIN;
+}
+/*! \brief minimum value of int32_t */
+template<>
+MSHADOW_XINLINE int MinValue<int32_t>(void) {
+ return INT_MIN;
+}
+/*! \brief minimum value of int64_t */
+template<>
+MSHADOW_XINLINE int64_t MinValue<int64_t>(void) {
+ return LLONG_MIN;
+}
+
+/*!
+ * \brief maximum value of certain types
+ * \tparam DType data type
+ */
+template<typename DType>
+MSHADOW_XINLINE DType MaxValue(void);
+/*! \brief maximum value of float */
+template<>
+MSHADOW_XINLINE float MaxValue<float>(void) {
+ return FLT_MAX;
+}
+/*! \brief maximum value of double */
+template<>
+MSHADOW_XINLINE double MaxValue<double>(void) {
+ return DBL_MAX;
+}
+/*! \brief maximum value of half */
+template<>
+MSHADOW_XINLINE half::half_t MaxValue<half::half_t>(void) {
+ return MSHADOW_HALF_MAX;
+}
+/*! \brief maximum value of uint8_t */
+template<>
+MSHADOW_XINLINE uint8_t MaxValue<uint8_t>(void) {
+ return UCHAR_MAX;
+}
+/*! \brief maximum value of int8_t */
+template<>
+MSHADOW_XINLINE int8_t MaxValue<int8_t>(void) {
+ return SCHAR_MAX;
+}
+/*! \brief maximum value of int32_t */
+template<>
+MSHADOW_XINLINE int MaxValue<int32_t>(void) {
+ return INT_MAX;
+}
+/*! \brief maximum value of int64_t */
+template<>
+MSHADOW_XINLINE int64_t MaxValue<int64_t>(void) {
+ return LLONG_MAX;
+}
+} // namespace limits
+
+/*! \brief sum reducer */
+struct sum {
+ /*! \brief do reduction into dst */
+ template<typename DType>
+ MSHADOW_XINLINE static void Reduce(volatile DType& dst, volatile DType src) { // NOLINT(*)
+ dst += src;
+ }
+ /*! \brief do stable reduction into dst */
+ template<typename DType>
+ MSHADOW_XINLINE static void Reduce(volatile DType& dst, volatile DType src, volatile DType& residual) { // NOLINT(*)
+ DType y = src - residual;
+ DType t = dst + y;
+ residual = (t - dst) - y;
+ dst = t;
+ }
+ /*! \brief combine the results of two reducers */
+ template<typename DType>
+ MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& src_val) { // NOLINT(*)
+ Reduce(dst_val, src_val);
+ }
+ /*! \brief combine the results of two reducers */
+ template<typename DType>
+ MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& dst_residual, volatile DType& src_val, volatile DType& src_residual) { // NOLINT(*)
+ DType t1 = dst_val + src_val;
+ DType e = t1 - dst_val;
+ DType t2 = ((src_val - e) + (dst_val - (t1 - e))) + dst_residual + src_residual;
+ dst_val = t1 + t2;
+ dst_residual = t2 - (dst_val - t1);
+ }
+ /*! \brief finalize reduction */
+ template<typename DType>
+ MSHADOW_XINLINE static void Finalize(volatile DType& dst) {} // NOLINT(*)
+ /*! \brief finalize reduction */
+ template<typename DType>
+ MSHADOW_XINLINE static void Finalize(volatile DType& dst, volatile DType& residual) {} // NOLINT(*)
+ /*!
+ *\brief calculate gradient of redres with respect to redsrc,
+ * redres: reduced result, redsrc: one of reduction element
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static DType PartialGrad(DType redres, DType redsrc) {
+ return 1;
+ }
+ /*!
+ *\brief set the initial value during reduction
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static void SetInitValue(DType &initv) { // NOLINT(*)
+ initv = 0;
+ }
+ /*!
+ *\brief set the initial value during reduction
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static void SetInitValue(DType &initv, DType &residual) { // NOLINT(*)
+ SetInitValue(initv);
+ residual = 0;
+ }
+};
+/*! \brief maximum reducer */
+struct maximum {
+ /*! \brief do reduction into dst */
+ template<typename DType>
+ MSHADOW_XINLINE static void Reduce(volatile DType& dst, volatile DType src) { // NOLINT(*)
+ using namespace std;
+#ifdef __CUDACC__
+ dst = ::max(dst, src);
+#else
+ dst = max(dst, src);
+#endif // __CUDACC__
+ }
+ /*! \brief do reduction into dst */
+ template<typename DType>
+ MSHADOW_XINLINE static void Reduce(volatile DType& dst, volatile DType src, volatile DType &none) { // NOLINT(*)
+ Reduce(dst, src);
+ }
+ /*! \brief combine the results of two reducers */
+ template<typename DType>
+ MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& src_val) { // NOLINT(*)
+ Reduce(dst_val, src_val);
+ }
+ /*! \brief combine the results of two reducers */
+ template<typename DType>
+ MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& dst_residual, volatile DType& src_val, volatile DType& src_residual) { // NOLINT(*)
+ Reduce(dst_val, src_val);
+ }
+ /*! \brief finalize reduction */
+ template<typename DType>
+ MSHADOW_XINLINE static void Finalize(volatile DType& dst) {} // NOLINT(*)
+ /*! \brief finalize reduction */
+ template<typename DType>
+ MSHADOW_XINLINE static void Finalize(volatile DType& dst, volatile DType& residual) {} // NOLINT(*)
+ /*!
+ * \brief calculate gradient of redres with respect to redsrc,
+ * redres: reduced result, redsrc: one of reduction element
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static DType PartialGrad(DType redres, DType redsrc) {
+ return redres == redsrc ? 1: 0;
+ }
+ /*!
+ *\brief set the initial value during reduction
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static void SetInitValue(DType &initv) { // NOLINT(*)
+ initv = limits::MinValue<DType>();
+ }
+ /*!
+ *\brief set the initial value during reduction
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static void SetInitValue(DType &initv, DType &none) { // NOLINT(*)
+ SetInitValue(initv);
+ }
+};
+/*! \brief minimum reducer */
+struct minimum {
+ /*! \brief do reduction into dst */
+ template<typename DType>
+ MSHADOW_XINLINE static void Reduce(volatile DType& dst, volatile DType src) { // NOLINT(*)
+ using namespace std;
+#ifdef __CUDACC__
+ dst = ::min(dst, src);
+#else
+ dst = min(dst, src);
+#endif // __CUDACC__
+ }
+ /*! \brief do reduction into dst */
+ template<typename DType>
+ MSHADOW_XINLINE static void Reduce(volatile DType& dst, volatile DType src, volatile DType &none) { // NOLINT(*)
+ Reduce(dst, src);
+ }
+ /*! \brief combine the results of two reducers */
+ template<typename DType>
+ MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& src_val) { // NOLINT(*)
+ Reduce(dst_val, src_val);
+ }
+ /*! \brief combine the results of two reducers */
+ template<typename DType>
+ MSHADOW_XINLINE static void Merge(volatile DType& dst_val, volatile DType& dst_residual, volatile DType& src_val, volatile DType& src_residual) { // NOLINT(*)
+ Reduce(dst_val, src_val);
+ }
+ /*! \brief finalize reduction */
+ template<typename DType>
+ MSHADOW_XINLINE static void Finalize(volatile DType& dst) {} // NOLINT(*)
+ /*! \brief finalize reduction */
+ template<typename DType>
+ MSHADOW_XINLINE static void Finalize(volatile DType& dst, volatile DType& residual) {} // NOLINT(*)
+ /*!
+ * \brief calculate gradient of redres with respect to redsrc,
+ * redres: reduced result, redsrc: one of reduction element
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static DType PartialGrad(DType redres, DType redsrc) {
+ return redres == redsrc ? 1: 0;
+ }
+ /*!
+ *\brief set the initial value during reduction
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static void SetInitValue(DType &initv) { // NOLINT(*)
+ initv = limits::MaxValue<DType>();
+ }
+ /*!
+ *\brief set the initial value during reduction
+ */
+ template<typename DType>
+ MSHADOW_XINLINE static void SetInitValue(DType &initv, DType &none) { // NOLINT(*)
+ SetInitValue(initv);
+ }
+};
+} // namespace red
+
+#define MSHADOW_TYPE_SWITCH(type, DType, ...) \
+ switch (type) { \
+ case mshadow::kFloat32: \
+ { \
+ typedef float DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat64: \
+ { \
+ typedef double DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat16: \
+ { \
+ typedef mshadow::half::half_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kUint8: \
+ { \
+ typedef uint8_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kInt8: \
+ { \
+ typedef int8_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kInt32: \
+ { \
+ typedef int32_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kInt64: \
+ { \
+ typedef int64_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ default: \
+ LOG(FATAL) << "Unknown type enum " << type; \
+ }
+
+#define MSHADOW_TYPE_SWITCH_WITH_HALF2(type, DType, ...) \
+ switch (type) { \
+ case mshadow::kFloat32: \
+ { \
+ typedef float DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat64: \
+ { \
+ typedef double DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat16: \
+ { \
+ typedef mshadow::half::half2_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kUint8: \
+ { \
+ typedef uint8_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kInt32: \
+ { \
+ typedef int32_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kInt64: \
+ { \
+ typedef int64_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ default: \
+ LOG(FATAL) << "Unknown type enum " << type; \
+ }
+
+#define MSHADOW_SGL_DBL_TYPE_SWITCH(type, DType, ...) \
+ switch (type) { \
+ case mshadow::kFloat32: \
+ { \
+ typedef float DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat64: \
+ { \
+ typedef double DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ default: \
+ LOG(FATAL) << "This operation only supports " \
+ "32-bit and 64-bit floating point"; \
+ }
+
+#define MSHADOW_REAL_TYPE_SWITCH(type, DType, ...) \
+ switch (type) { \
+ case mshadow::kFloat32: \
+ { \
+ typedef float DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat64: \
+ { \
+ typedef double DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat16: \
+ { \
+ typedef mshadow::half::half_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kUint8: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types not uint8"; \
+ break; \
+ case mshadow::kInt8: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types not int8"; \
+ break; \
+ case mshadow::kInt32: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types, not int32";\
+ break; \
+ case mshadow::kInt64: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types, not int64";\
+ break; \
+ default: \
+ LOG(FATAL) << "Unknown type enum " << type; \
+ }
+
+#define MSHADOW_REAL_TYPE_SWITCH_EX(type$, DType$, DLargeType$, ...) \
+ switch (type$) { \
+ case mshadow::kFloat32: \
+ { \
+ typedef float DType$; \
+ typedef float DLargeType$; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat64: \
+ { \
+ typedef double DType$; \
+ typedef double DLargeType$; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kFloat16: \
+ { \
+ typedef mshadow::half::half_t DType$; \
+ typedef float DLargeType$; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kUint8: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types not uint8"; \
+ break; \
+ case mshadow::kInt8: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types not int8"; \
+ break; \
+ case mshadow::kInt32: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types, not int32";\
+ break; \
+ case mshadow::kInt64: \
+ LOG(FATAL) << "This operation only support " \
+ "floating point types, not int64";\
+ break; \
+ default: \
+ LOG(FATAL) << "Unknown type enum " << type$; \
+ }
+
+#define MSHADOW_LAYOUT_SWITCH(layout, Layout, ...) \
+ switch (layout) { \
+ case mshadow::kNCHW: \
+ { \
+ const int Layout = kNCHW; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kNHWC: \
+ { \
+ const int Layout = kNHWC; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kNCDHW: \
+ { \
+ const int Layout = kNCDHW; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ case mshadow::kNDHWC: \
+ { \
+ const int Layout = kNDHWC; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ default: \
+ LOG(FATAL) << "Unknown layout enum " << layout; \
+ }
+
+/*!
+ * \brief Only supports int64 index type for aux_data
+ * in NDArray class fow now.
+ */
+#define MSHADOW_IDX_TYPE_SWITCH(type, DType, ...) \
+ switch (type) { \
+ case mshadow::kInt64: \
+ { \
+ typedef int64_t DType; \
+ {__VA_ARGS__} \
+ } \
+ break; \
+ default: \
+ LOG(FATAL) << "Unknown type enum " << type; \
+ }
+
+/*! \brief get data type size from type enum */
+inline size_t mshadow_sizeof(int type) {
+ int size = 0;
+ MSHADOW_TYPE_SWITCH(type, DType, size = sizeof(DType););
+ return size;
+}
+
+} // namespace mshadow
+#endif // MSHADOW_BASE_H_
diff --git a/3rdparty/mshadow/mshadow/cuda/reduce.cuh b/3rdparty/mshadow/mshadow/cuda/reduce.cuh
new file mode 100644
index 0000000..921d5ad
--- /dev/null
+++ b/3rdparty/mshadow/mshadow/cuda/reduce.cuh
@@ -0,0 +1,120 @@
+/*!
+ * Copyright (c) 2014 by Contributors
+ * \file reduce.cuh
+ * \brief helper functions to do reduction
+ * \author Tianqi Chen
+ */
... 12545 lines suppressed ...