You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/11/13 13:19:03 UTC
[incubator-mxnet] branch master updated: Tool to ease compilation
and reproduction of test results (#13202)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new bdb4443 Tool to ease compilation and reproduction of test results (#13202)
bdb4443 is described below
commit bdb44438b8b4e401a59f2a3df1e3ef66df5ed523
Author: Pedro Larroy <92...@users.noreply.github.com>
AuthorDate: Tue Nov 13 14:18:45 2018 +0100
Tool to ease compilation and reproduction of test results (#13202)
* Add tool to simplify reproducing tests
* add local build
* Add cmake_options.yaml
* minor
* Fix license
* Fix licenses
* Rename file, address CR comments about gpu build function
* Address Marco's comments
---
cmake/cmake_options.yml | 49 +++++++++++++++
dev_menu.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 203 insertions(+)
diff --git a/cmake/cmake_options.yml b/cmake/cmake_options.yml
new file mode 100644
index 0000000..6fbf4e1
--- /dev/null
+++ b/cmake/cmake_options.yml
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+--- # CMake configuration
+USE_CUDA: OFF # Build with CUDA support
+USE_OLDCMAKECUDA: OFF # Build with old cmake cuda
+USE_NCCL: OFF # Use NVidia NCCL with CUDA
+USE_OPENCV: ON # Build with OpenCV support
+USE_OPENMP: ON # Build with Openmp support
+USE_CUDNN: ON # Build with cudnn support) # one could set CUDNN_ROOT for search path
+USE_SSE: ON # Build with x86 SSE instruction support IF NOT ARM
+USE_F16C: ON # Build with x86 F16C instruction support) # autodetects support if ON
+USE_LAPACK: ON # Build with lapack support
+USE_MKL_IF_AVAILABLE: ON # Use MKL if found
+USE_MKLML_MKL: ON # Use MKLDNN variant of MKL (if MKL found) IF USE_MKL_IF_AVAILABLE AND (NOT APPLE)
+USE_MKLDNN: ON # Use MKLDNN variant of MKL (if MKL found) IF USE_MKL_IF_AVAILABLE AND (NOT APPLE)
+USE_OPERATOR_TUNING: ON # Enable auto-tuning of operators IF NOT MSVC
+USE_GPERFTOOLS: ON # Build with GPerfTools support (if found)
+USE_JEMALLOC: ON # Build with Jemalloc support
+USE_PROFILER: ON # Build with Profiler support
+USE_DIST_KVSTORE: OFF # Build with DIST_KVSTORE support
+USE_PLUGINS_WARPCTC: OFF # Use WARPCTC Plugins
+USE_PLUGIN_CAFFE: OFF # Use Caffe Plugin
+USE_CPP_PACKAGE: OFF # Build C++ Package
+USE_MXNET_LIB_NAMING: ON # Use MXNet library naming conventions.
+USE_GPROF: OFF # Compile with gprof (profiling) flag
+USE_CXX14_IF_AVAILABLE: OFF # Build with C++14 if the compiler supports it
+USE_VTUNE: OFF # Enable use of Intel Amplifier XE (VTune)) # one could set VTUNE_ROOT for search path
+ENABLE_CUDA_RTC: ON # Build with CUDA runtime compilation support
+BUILD_CPP_EXAMPLES: ON # Build cpp examples
+INSTALL_EXAMPLES: OFF # Install the example source files.
+USE_SIGNAL_HANDLER: OFF # Print stack traces on segfaults.
+USE_TENSORRT: OFF # Enable infeference optimization with TensorRT.
+USE_ASAN: OFF # Enable Clang/GCC ASAN sanitizers.
+ENABLE_TESTCOVERAGE: OFF # Enable compilation with test coverage metric output
diff --git a/dev_menu.py b/dev_menu.py
new file mode 100755
index 0000000..fc80b0a
--- /dev/null
+++ b/dev_menu.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+"""Tool to ease working with the build system and reproducing test results"""
+
+import os
+import sys
+from subprocess import check_call
+import shlex
+from ci.util import retry, remember_cwd
+from typing import List
+from collections import OrderedDict
+import logging
+import yaml
+
+class Confirm(object):
+ def __init__(self, cmds):
+ self.cmds = cmds
+
+ def __call__(self):
+ resp = input("This will run the following command(s) '{}' are you sure? yes / no: ".format(self.cmds))
+ while True:
+ if resp.lower() == 'yes':
+ handle_commands(self.cmds)
+ return
+ elif resp.lower() == 'no':
+ return
+ else:
+ resp = input("Please answer yes or no: ")
+
+class CMake(object):
+ def __init__(self, cmake_options_yaml='cmake/cmake_options.yml'):
+ self.cmake_options_yaml = cmake_options_yaml
+ self.cmake_options = None
+ self.read_config()
+
+ def read_config(self):
+ assert os.path.isfile(self.cmake_options_yaml)
+ with open(self.cmake_options_yaml, 'r') as f:
+ self.cmake_options = yaml.load(f)
+
+ def _cmdlineflags(self):
+ res = []
+ def _bool_ON_OFF(x):
+ if x:
+ return 'ON'
+ else:
+ return 'OFF'
+ for opt,v in self.cmake_options.items():
+ res.append('-D{}={}'.format(opt,_bool_ON_OFF(v)))
+ return res
+
+ def cmake_command(self) -> str:
+ """
+ :return: Cmake command to run given the options
+ """
+ cmd_lst = ['cmake']
+ cmd_lst.extend(self._cmdlineflags())
+ return cmd_lst
+
+ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):
+ logging.info("CMake / {} build in directory {}".format(
+ generator, os.path.abspath(build_dir)))
+ cmd_lst = self.cmake_command()
+ os.makedirs(build_dir, exist_ok=True)
+ with remember_cwd():
+ os.chdir(build_dir)
+ cmd_lst.extend(['-G{}'.format(generator), '..'])
+ logging.info('Executing: {}'.format('\t\n'.join(cmd_lst)))
+ check_call(cmd_lst)
+ logging.info('Now building')
+ check_call(shlex.split(build_cmd))
+
+
+
+COMMANDS = OrderedDict([
+ ('[Docker] sanity_check',
+ "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"),
+ ('[Docker] Python3 CPU unittests',
+ [
+ "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas",
+ "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu",
+ ]),
+ ('[Docker] Python3 GPU unittests',
+ [
+ "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu",
+ "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu",
+ ]),
+ ('[Local] CMake build (using cmake/cmake_options.yaml)',
+ CMake()),
+ ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)',
+ Confirm("ci/docker/runtime_functions.sh clean_repo"))
+])
+
+def clip(x, mini, maxi):
+ return min(max(x,mini), maxi)
+
+@retry((ValueError, RuntimeError), 3, delay_s = 0)
+def show_menu(items: List[str], header=None) -> int:
+ def hr():
+ print(''.join(['-']*30))
+ if header:
+ print(header)
+ hr()
+ for i,x in enumerate(items,1):
+ print('{}. {}'.format(i,x))
+ hr()
+ choice = int(input('Choose option> ')) - 1
+ if choice < 0 or choice >= len(items):
+ raise RuntimeError('Choice must be between {} and {}'.format(1, len(items)))
+ return choice
+
+def handle_commands(cmds) -> None:
+ def handle_command(cmd):
+ logging.info("Executing command: %s",cmd)
+ check_call(shlex.split(cmd))
+
+ if type(cmds) is list:
+ for cmd in cmds:
+ handle_commands(cmd)
+ elif type(cmds) is str:
+ handle_command(cmds)
+ elif callable(cmds):
+ cmds()
+ else:
+ raise RuntimeError("handle_commands(cmds): argument should be str or List[str] but is %s", type(cmds))
+
+def main():
+ logging.getLogger().setLevel(logging.INFO)
+ command_list = list(COMMANDS.keys())
+ choice = show_menu(command_list, 'Available actions')
+ handle_commands(COMMANDS[command_list[choice]])
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())
+