[GitHub] marcoabreu closed pull request #13529: Improve dev_menu usability, local build and virtualenv

[GitBox <gi...@apache.org>]

marcoabreu closed pull request #13529: Improve dev_menu usability, local build and virtualenv
URL: https://github.com/apache/incubator-mxnet/pull/13529
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/.gitignore b/.gitignore
index c8a813649bb..c7530ab69c6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -177,3 +177,5 @@ cxx
 *.gcno
 coverage.xml
 
+# Local CMake build config
+cmake_options.yml
diff --git a/cmake/cmake_options.yml b/cmake/cmake_options.yml
index 01446f7b8f2..a4323feb92d 100644
--- a/cmake/cmake_options.yml
+++ b/cmake/cmake_options.yml
@@ -16,7 +16,7 @@
 # under the License.
 
 --- # CMake configuration
-USE_CUDA: "ON" # Build with CUDA support
+USE_CUDA: "OFF" # Build with CUDA support
 USE_OLDCMAKECUDA: "OFF" # Build with old cmake cuda
 USE_NCCL: "OFF" # Use NVidia NCCL with CUDA
 USE_OPENCV: "ON" # Build with OpenCV support
@@ -48,3 +48,6 @@ USE_TENSORRT: "OFF" # Enable infeference optimization with TensorRT.
 USE_ASAN: "OFF" # Enable Clang/GCC ASAN sanitizers.
 ENABLE_TESTCOVERAGE: "OFF" # Enable compilation with test coverage metric output
 CMAKE_BUILD_TYPE: "Debug"
+CMAKE_CUDA_COMPILER_LAUNCHER: "ccache"
+CMAKE_C_COMPILER_LAUNCHER: "ccache"
+CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
diff --git a/dev_menu.py b/dev_menu.py
index 0fd78cb222e..ebffe14f762 100755
--- a/dev_menu.py
+++ b/dev_menu.py
@@ -20,6 +20,7 @@
 # -*- coding: utf-8 -*-
 """Tool to ease working with the build system and reproducing test results"""
 
+import argparse
 import os
 import sys
 from subprocess import check_call
@@ -29,6 +30,11 @@
 from collections import OrderedDict
 import logging
 import yaml
+import shutil
+
+DEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv')
+DEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3')
+DEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml')
 
 class Confirm(object):
     def __init__(self, cmds):
@@ -46,7 +52,7 @@ def __call__(self):
                 resp = input("Please answer yes or no: ")
 
 class CMake(object):
-    def __init__(self, cmake_options_yaml='cmake_options.yml', cmake_options_yaml_default='cmake/cmake_options.yml'):
+    def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS, cmake_options_yaml_default='cmake/cmake_options.yml'):
         if os.path.exists(cmake_options_yaml):
             self.cmake_options_yaml = cmake_options_yaml
         else:
@@ -87,10 +93,29 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):
             logging.info('Now building')
             check_call(shlex.split(build_cmd))
 
-
+def create_virtualenv(venv_exe, pyexe, venv) -> None:
+    logging.info("Creating virtualenv in %s with python %s", venv, pyexe)
+    if not (venv_exe and pyexe and venv):
+        logging.warn("Skipping creation of virtualenv")
+        return
+    check_call([venv_exe, '-p', pyexe, venv])
+    activate_this_py = os.path.join(venv, 'bin', 'activate_this.py')
+    # Activate virtualenv in this interpreter
+    exec(open(activate_this_py).read(), dict(__file__=activate_this_py))
+    check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python'])
+    check_call(['pip', 'install', '-r', 'tests/requirements.txt'])
+
+def create_virtualenv_default():
+    create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV)
+    logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV)
 
 COMMANDS = OrderedDict([
-    ('[Docker] sanity_check',
+    ('[Local build] CMake/Ninja build (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) (creates {} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV),
+    [
+        CMake(),
+        create_virtualenv_default,
+    ]),
+    ('[Docker] sanity_check. Check for linting and code formatting.',
         "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"),
     ('[Docker] Python3 CPU unittests',
     [
@@ -117,8 +142,6 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):
         "ci/build.py -p armv7",
         "ci/build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
     ]),
-    ('[Local] CMake build (using cmake/cmake_options.yaml)',
-        CMake()),
     ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)',
        Confirm("ci/docker/runtime_functions.sh clean_repo"))
 ])
@@ -128,6 +151,7 @@ def clip(x, mini, maxi):
 
 @retry((ValueError, RuntimeError), 3, delay_s = 0)
 def show_menu(items: List[str], header=None) -> int:
+    print('\n-- MXNet dev menu --\n')
     def hr():
         print(''.join(['-']*30))
     if header:
@@ -156,11 +180,47 @@ def handle_command(cmd):
     else:
         raise RuntimeError("handle_commands(cmds): argument should be str or List[str] but is %s", type(cmds))
 
-def main():
-    logging.getLogger().setLevel(logging.INFO)
+def use_menu_ui(args) -> None:
     command_list = list(COMMANDS.keys())
     choice = show_menu(command_list, 'Available actions')
     handle_commands(COMMANDS[command_list[choice]])
+
+def build(args) -> None:
+    """Build using CMake"""
+    venv_exe = shutil.which('virtualenv')
+    pyexe = shutil.which(args.pyexe)
+    if not venv_exe:
+        logging.warn("virtualenv wasn't found in path, it's recommended to install virutalenv to manage python environments")
+    if not pyexe:
+        logging.warn("Python executable %s not found in path", args.pyexe)
+    if args.cmake_options:
+        cmake = CMake(args.cmake_options)
+    else:
+        cmake = CMake()
+    cmake()
+    create_virtualenv(venv_exe, pyexe, args.venv)
+
+def main():
+    logging.getLogger().setLevel(logging.INFO)
+    parser = argparse.ArgumentParser(description="""Utility for compiling and testing MXNet easily""")
+    parser.set_defaults(command='use_menu_ui')
+
+    subparsers = parser.add_subparsers(help='sub-command help')
+    build_parser = subparsers.add_parser('build', help='build with the specified flags from file')
+    build_parser.add_argument('cmake_options', nargs='?',
+        help='File containing CMake options in YAML')
+    build_parser.add_argument('-v', '--venv',
+        type=str,
+        default=DEFAULT_PYENV,
+        help='virtualenv dir')
+    build_parser.add_argument('-p', '--pyexe',
+        type=str,
+        default=DEFAULT_PYTHON,
+        help='python executable')
+
+    build_parser.set_defaults(command='build')
+    args = parser.parse_args()
+    globals()[args.command](args)
     return 0
 
 if __name__ == '__main__':
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 3ca696b288c..f64f7ffb670 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,3 +2,4 @@
 mock
 nose
 nose-timer
+ipython


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services