You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/10/22 18:47:45 UTC
[GitHub] marcoabreu closed pull request #12094: [MXNET-793] ★ Virtualized testing in CI with QEMU ★
marcoabreu closed pull request #12094: [MXNET-793] ★ Virtualized testing in CI with QEMU ★
URL: https://github.com/apache/incubator-mxnet/pull/12094
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/README.md b/ci/README.md
index 69308756943..3737fc74065 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -90,3 +90,13 @@ For all builds a directory from the host system is mapped where ccache will stor
compiled object files (defaults to /tmp/ci_ccache). This will speed up rebuilds
significantly. You can set this directory explicitly by setting CCACHE_DIR environment
variable. All ccache instances are currently set to be 10 Gigabytes max in size.
+
+
+## Testing with QEMU
+To run the unit tests under qemu:
+```
+./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu
+```
+
+To get a shell on the container and debug issues with the emulator itself:
+Run the output of `./build.py -p test.arm_qemu --print-docker-run`
diff --git a/ci/build.py b/ci/build.py
index b7c86adda71..e2554d9b8ce 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -394,7 +394,7 @@ def main() -> int:
help="platform",
type=str)
- parser.add_argument("--build-only",
+ parser.add_argument("-b", "--build-only",
help="Only build the container, don't build the project",
action='store_true')
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.test.arm_qemu b/ci/docker/Dockerfile.build.test.arm_qemu
new file mode 100644
index 00000000000..fde105c3f98
--- /dev/null
+++ b/ci/docker/Dockerfile.build.test.arm_qemu
@@ -0,0 +1,44 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work
+
+RUN apt-get update
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_arm_qemu.sh /work
+RUN /work/ubuntu_arm_qemu.sh
+
+COPY install/ubuntu_arm_qemu_bin.sh /work
+RUN /work/ubuntu_arm_qemu_bin.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+COPY qemu/* /work/
+COPY qemu/ansible.cfg /etc/ansible/ansible.cfg
+
+CMD ["./runtime_functions.py","run_ut_py3_qemu"]
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat
old mode 100755
new mode 100644
diff --git a/ci/docker/install/ubuntu_arm_qemu.sh b/ci/docker/install/ubuntu_arm_qemu.sh
new file mode 100755
index 00000000000..c30dc4f13d1
--- /dev/null
+++ b/ci/docker/install/ubuntu_arm_qemu.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -exuo pipefail
+
+apt-get install -y \
+ cmake \
+ curl \
+ wget \
+ git \
+ qemu \
+ qemu-system-arm \
+ unzip \
+ bzip2 \
+ vim-nox
+
+pip3 install ansible ipython
diff --git a/ci/docker/install/ubuntu_arm_qemu_bin.sh b/ci/docker/install/ubuntu_arm_qemu_bin.sh
new file mode 100755
index 00000000000..d4f81185c16
--- /dev/null
+++ b/ci/docker/install/ubuntu_arm_qemu_bin.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -exuo pipefail
+
+#
+# This disk image and kernels for virtual testing with QEMU is generated with some manual OS
+# installation steps with the scripts and documentation found in the ci/qemu/ folder.
+#
+# The image has a base Debian OS and MXNet runtime dependencies installed.
+# The root password is empty and there's a "qemu" user without password. SSH access is enabled as
+# well.
+#
+# See also: ci/qemu/README.md
+#
+
+REMOTE="https://s3-us-west-2.amazonaws.com/mxnet-ci-prod-slave-data"
+curl -f ${REMOTE}/vda_debian_stretch.qcow2.bz2 | bunzip2 > vda.qcow2
+curl -f ${REMOTE}/vmlinuz -o vmlinuz
+curl -f ${REMOTE}/initrd.img -o initrd.img
+
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh
index a6051638665..d6e66aa45c3 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python.sh
@@ -22,7 +22,7 @@
set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python-dev python3-dev virtualenv wget
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
wget -nv https://bootstrap.pypa.io/get-pip.py
diff --git a/ci/docker/qemu/README.md b/ci/docker/qemu/README.md
new file mode 100644
index 00000000000..1dcaa5aeb60
--- /dev/null
+++ b/ci/docker/qemu/README.md
@@ -0,0 +1 @@
+These are files used in the docker container that runs QEMU
diff --git a/ci/docker/qemu/ansible.cfg b/ci/docker/qemu/ansible.cfg
new file mode 100644
index 00000000000..24e2ec87b94
--- /dev/null
+++ b/ci/docker/qemu/ansible.cfg
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[defaults]
+host_key_checking = False
+ansible_python_interpreter = /usr/bin/python3
diff --git a/ci/docker/qemu/playbook.yml b/ci/docker/qemu/playbook.yml
new file mode 100644
index 00000000000..3b9e7c52bde
--- /dev/null
+++ b/ci/docker/qemu/playbook.yml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+- name: provision QEMU VM
+ hosts: all
+ gather_facts: no
+ become: true
+ become_user: root
+ tasks:
+ - name: Wait until ssh is available
+ wait_for_connection:
+ delay: 0
+ sleep: 3
+ timeout: 400
+ - command: hostname
+ register: h
+ - debug: msg="{{ h.stdout }}"
+
+ - name: copy mxnet artifacts
+ copy:
+ src: "{{ item }}"
+ dest: mxnet_dist/
+ with_fileglob: "/work/mxnet/build/*.whl"
+
+ - name: copy runtime_functions.py
+ copy:
+ src: "/work/runtime_functions.py"
+ dest: .
+ - file:
+ path: runtime_functions.py
+ mode: 0755
+
+
diff --git a/ci/docker/qemu/qemu_run.sh b/ci/docker/qemu/qemu_run.sh
new file mode 100755
index 00000000000..53a6487aa0d
--- /dev/null
+++ b/ci/docker/qemu/qemu_run.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -exuo pipefail
+
+qemu-system-arm -M virt -m 1024 \
+ -kernel vmlinuz \
+ -initrd initrd.img \
+ -append 'root=/dev/vda1' \
+ -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
+ -device virtio-blk-device,drive=hd \
+ -netdev user,id=mynet,hostfwd=tcp::2222-:22 \
+ -device virtio-net-device,netdev=mynet \
+ -nographic \
+ -display none
diff --git a/ci/docker/qemu/runtime_functions.py b/ci/docker/qemu/runtime_functions.py
new file mode 100755
index 00000000000..6cf01b6f912
--- /dev/null
+++ b/ci/docker/qemu/runtime_functions.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+"""Runtime functions to use in docker / testing"""
+
+__author__ = 'Pedro Larroy'
+__version__ = '0.1'
+
+import os
+import sys
+import subprocess
+import argparse
+import logging
+from subprocess import call, check_call, Popen, DEVNULL, PIPE
+import time
+import sys
+import types
+import glob
+
+def activate_this(base):
+ import site
+ import os
+ import sys
+ if sys.platform == 'win32':
+ site_packages = os.path.join(base, 'Lib', 'site-packages')
+ else:
+ site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages')
+ prev_sys_path = list(sys.path)
+ sys.real_prefix = sys.prefix
+ sys.prefix = base
+ # Move the added items to the front of the path:
+ new_sys_path = []
+ for item in list(sys.path):
+ if item not in prev_sys_path:
+ new_sys_path.append(item)
+ sys.path.remove(item)
+ sys.path[:0] = new_sys_path
+
+def run_ut_py3_qemu():
+ from vmcontrol import VM
+ with VM() as vm:
+ logging.info("VM provisioning with ansible")
+ check_call(["ansible-playbook", "-v", "-u", "qemu", "-i", "localhost:{},".format(vm.ssh_port), "playbook.yml"])
+ logging.info("VM provisioned successfully.")
+ logging.info("sync tests")
+ check_call(['rsync', '-e', 'ssh -p{}'.format(vm.ssh_port), '-a', 'mxnet/tests', 'qemu@localhost:mxnet'])
+ logging.info("execute tests")
+ check_call(["ssh", "-o", "ServerAliveInterval=5", "-p{}".format(vm.ssh_port), "qemu@localhost", "./runtime_functions.py", "run_ut_python3_qemu_internal"])
+ logging.info("tests finished, vm shutdown.")
+ vm.shutdown()
+
+def run_ut_python3_qemu_internal():
+ """this runs inside the vm, it's run by the playbook above by ansible"""
+ pkg = glob.glob('mxnet_dist/*.whl')[0]
+ logging.info("=== NOW Running inside QEMU ===")
+ logging.info("PIP Installing %s", pkg)
+ check_call(['sudo', 'pip3', 'install', pkg])
+ logging.info("PIP Installing mxnet/tests/requirements.txt")
+ check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/tests/requirements.txt'])
+ logging.info("Running tests in mxnet/tests/python/unittest/")
+ check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+
+
+def parsed_args():
+ parser = argparse.ArgumentParser(description="""python runtime functions""", epilog="")
+ parser.add_argument('command',nargs='*',
+ help="Name of the function to run with arguments")
+ args = parser.parse_args()
+ return (args, parser)
+
+def script_name() -> str:
+ return os.path.split(sys.argv[0])[1]
+
+def chdir_to_script_directory():
+ # We need to be in the same directory than the script so the commands in the dockerfiles work as
+ # expected. But the script can be invoked from a different path
+ base = os.path.split(os.path.realpath(__file__))[0]
+ os.chdir(base)
+
+def main():
+ logging.getLogger().setLevel(logging.DEBUG)
+ logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
+ chdir_to_script_directory()
+
+ # Run function with name passed as argument
+ (args, parser) = parsed_args()
+ logging.info("%s", args.command)
+ if args.command:
+ fargs = args.command[1:]
+ globals()[args.command[0]](*fargs)
+ return 0
+ else:
+ parser.print_help()
+ fnames = [x for x in globals() if type(globals()[x]) is types.FunctionType]
+ print('\nAvailable functions: {}'.format(' '.join(fnames)))
+ return 1
+
+if __name__ == '__main__':
+ sys.exit(main())
+
diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py
new file mode 100644
index 00000000000..2262bc77bf9
--- /dev/null
+++ b/ci/docker/qemu/vmcontrol.py
@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+"""Utilities to control a guest VM, used for virtual testing with QEMU"""
+
+__author__ = 'Pedro Larroy'
+__version__ = '0.1'
+
+import os
+import sys
+import subprocess
+import argparse
+import logging
+from subprocess import call, check_call, Popen, DEVNULL, PIPE
+import time
+import sys
+import multiprocessing
+import shlex
+
+###################################################
+#
+# Virtual testing with QEMU
+#
+# We start QEMU instances that have a local port in the host redirected to the ssh port.
+#
+# The VMs are provisioned after boot, tests are run and then they are stopped
+#
+
+QEMU_RUN="""
+qemu-system-arm -M virt -m {ram} \
+ -kernel vmlinuz \
+ -initrd initrd.img \
+ -append 'root=/dev/vda1' \
+ -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
+ -device virtio-blk-device,drive=hd \
+ -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \
+ -device virtio-net-device,netdev=mynet \
+ -display none -nographic
+"""
+
+class VMError(RuntimeError):
+ pass
+
+class VM:
+ """Control of the virtual machine"""
+ def __init__(self, ssh_port=2222):
+ self.log = logging.getLogger(VM.__name__)
+ self.ssh_port = ssh_port
+ self.timeout_s = 300
+ self.qemu_process = None
+ self._detach = False
+
+ def __enter__(self):
+ self.start()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if not self._detach:
+ self.shutdown()
+ self.terminate()
+
+ def start(self):
+ self.log.info("Starting VM, ssh port redirected to localhost:%s", self.ssh_port)
+ if self.is_running():
+ raise VMError("VM is running, shutdown first")
+ self.qemu_process = run_qemu(self.ssh_port)
+ def keep_waiting():
+ return self.is_running()
+
+ ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting, self.timeout_s)
+
+ if not self.is_running():
+ (_, stderr) = self.qemu_process.communicate()
+ raise VMError("VM failed to start, retcode: {}, stderr: {}".format( self.retcode(), stderr.decode()))
+
+ if not ssh_working:
+ if self.is_running():
+ self.log.error("VM running but SSH is not working")
+ self.terminate()
+ raise VMError("SSH is not working after {} seconds".format(self.timeout_s))
+ self.log.info("VM is online and SSH is up")
+
+ def is_running(self):
+ return self.qemu_process and self.qemu_process.poll() is None
+
+ def retcode(self):
+ if self.qemu_process:
+ return self.qemu_process.poll()
+ else:
+ raise RuntimeError('qemu process was not started')
+
+ def terminate(self):
+ if self.qemu_process:
+ logging.info("send term signal")
+ self.qemu_process.terminate()
+ time.sleep(3)
+ logging.info("send kill signal")
+ self.qemu_process.kill()
+ self.qemu_process.wait()
+ self.qemu_process = None
+ else:
+ logging.warn("VM.terminate: QEMU process not running")
+
+ def detach(self):
+ self._detach = True
+
+ def shutdown(self):
+ if self.qemu_process:
+ logging.info("Shutdown via ssh")
+ # ssh connection will be closed with an error
+ call(["ssh", "-o", "StrictHostKeyChecking=no", "-p", str(self.ssh_port), "qemu@localhost",
+ "sudo", "poweroff"])
+ ret = self.qemu_process.wait(timeout=90)
+ self.log.info("VM on port %s has shutdown (exit code %d)", self.ssh_port, ret)
+ self.qemu_process = None
+
+ def wait(self):
+ if self.qemu_process:
+ self.qemu_process.wait()
+
+ def __del__(self):
+ if self.is_running and not self._detach:
+ logging.info("VM destructor hit")
+ self.terminate()
+
+def run_qemu(ssh_port=2222):
+ cmd = QEMU_RUN.format(ssh_port=ssh_port, ram=4096)
+ logging.info("QEMU command: %s", cmd)
+ qemu_process = Popen(shlex.split(cmd), stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE)
+ return qemu_process
+
+
+def wait_ssh_open(server, port, keep_waiting=None, timeout=None):
+ """ Wait for network service to appear
+ @param server: host to connect to (str)
+ @param port: port (int)
+ @param timeout: in seconds, if None or 0 wait forever
+ @return: True of False, if timeout is None may return only True or
+ throw unhandled network exception
+ """
+ import socket
+ import errno
+ import time
+ log = logging.getLogger('wait_ssh_open')
+ sleep_s = 0
+ if timeout:
+ from time import time as now
+ # time module is needed to calc timeout shared between two exceptions
+ end = now() + timeout
+
+ while True:
+ log.debug("Sleeping for %s second(s)", sleep_s)
+ time.sleep(sleep_s)
+ s = socket.socket()
+ try:
+ if keep_waiting and not keep_waiting():
+ log.debug("keep_waiting() is set and evaluates to False")
+ return False
+
+ if timeout:
+ next_timeout = end - now()
+ if next_timeout < 0:
+ log.debug("connect time out")
+ return False
+ else:
+ log.debug("connect timeout %d s", next_timeout)
+ s.settimeout(next_timeout)
+
+ log.info("connect %s:%d", server, port)
+ s.connect((server, port))
+ ret = s.recv(1024).decode()
+ if ret and ret.startswith('SSH'):
+ s.close()
+ log.info("wait_ssh_open: port %s:%s is open and ssh is ready", server, port)
+ return True
+ else:
+ log.debug("Didn't get the SSH banner")
+ s.close()
+
+ except ConnectionError as err:
+ log.debug("ConnectionError %s", err)
+ if sleep_s == 0:
+ sleep_s = 1
+ else:
+ sleep_s *= 2
+
+ except socket.gaierror as err:
+ log.debug("gaierror %s",err)
+ return False
+
+ except socket.timeout as err:
+ # this exception occurs only if timeout is set
+ if timeout:
+ return False
+
+ except TimeoutError as err:
+ # catch timeout exception from underlying network library
+ # this one is different from socket.timeout
+ raise
+
+
+def wait_port_open(server, port, timeout=None):
+ """ Wait for network service to appear
+ @param server: host to connect to (str)
+ @param port: port (int)
+ @param timeout: in seconds, if None or 0 wait forever
+ @return: True of False, if timeout is None may return only True or
+ throw unhandled network exception
+ """
+ import socket
+ import errno
+ import time
+ sleep_s = 0
+ if timeout:
+ from time import time as now
+ # time module is needed to calc timeout shared between two exceptions
+ end = now() + timeout
+
+ while True:
+ logging.debug("Sleeping for %s second(s)", sleep_s)
+ time.sleep(sleep_s)
+ s = socket.socket()
+ try:
+ if timeout:
+ next_timeout = end - now()
+ if next_timeout < 0:
+ return False
+ else:
+ s.settimeout(next_timeout)
+
+ logging.info("connect %s %d", server, port)
+ s.connect((server, port))
+
+ except ConnectionError as err:
+ logging.debug("ConnectionError %s", err)
+ if sleep_s == 0:
+ sleep_s = 1
+
+ except socket.gaierror as err:
+ logging.debug("gaierror %s",err)
+ return False
+
+ except socket.timeout as err:
+ # this exception occurs only if timeout is set
+ if timeout:
+ return False
+
+ except TimeoutError as err:
+ # catch timeout exception from underlying network library
+ # this one is different from socket.timeout
+ raise
+
+ else:
+ s.close()
+ logging.info("wait_port_open: port %s:%s is open", server, port)
+ return True
+
diff --git a/ci/qemu/README.md b/ci/qemu/README.md
new file mode 100644
index 00000000000..6dde8916b28
--- /dev/null
+++ b/ci/qemu/README.md
@@ -0,0 +1,71 @@
+# QEMU base image creation
+
+This folder contains scripts and configuration to create a QEMU virtual drive with a debian system.
+
+The order of execution is:
+- `init.sh` to download the installation kernel and ramdisk
+- `preseed.sh` to preseed the debian installer so it doesn't ask questions
+- `copy.sh` to extract the kernel and ramdisk from the installed system
+- `run.sh` to boot the system and fine tune the image
+
+# Description of the process:
+
+# Preparing the base image
+
+First, an installation is made using installer kernel and initrd by using the scripts above.
+
+# After installation, we extract initrd and kernel from the installation drive
+
+The commands look like this:
+
+`virt-copy-out -a hda.qcow2 /boot/initrd.img-4.15.0-30-generic-lpae .`
+
+In the same way for the kernel.
+
+Then we install packages and dependencies on the qemu image:
+
+apt install -y sudo python3-dev virtualenv wget libgfortran3 libopenblas-base rsync build-essential
+libopenblas-dev libomp5
+
+We enable sudo and passwordless logins:
+
+Add file `/etc/sudoers.d/01-qemu`
+With content:
+```
+qemu ALL=(ALL) NOPASSWD: ALL
+```
+
+Edit: `/etc/ssh/sshd_config`
+
+And set the following options:
+```
+PermitEmptyPasswords yes
+PasswordAuthentication yes
+PermitRootLogin yes
+```
+
+Disable root and user passwords with `passwd -d`
+
+Edit ` /etc/pam.d/common-auth`
+
+Replace `auth [success=1 default=ignore] pam_unix.so nullok_secure` by
+```
+auth [success=1 default=ignore] pam_unix.so nullok
+```
+
+As root to install system wide:
+
+```
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+apt-get clean
+```
+
+Afterwards install mxnet python3 deps:
+
+```
+pip3 install -r mxnet_requirements.txt
+```
+
+
+To access qemu control console from tmux: `ctrl-a a c`
diff --git a/ci/qemu/copy.sh b/ci/qemu/copy.sh
new file mode 100755
index 00000000000..f39a9d08350
--- /dev/null
+++ b/ci/qemu/copy.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash -exuo pipefail
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Extract kernel from image
+
+set -ex
+virt-copy-out -a vda.qcow2 /boot/vmlinuz-3.16.0-6-armmp-lpae /boot/initrd.img-3.16.0-6-armmp-lpae .
diff --git a/ci/qemu/init.sh b/ci/qemu/init.sh
new file mode 100755
index 00000000000..1698cb10f27
--- /dev/null
+++ b/ci/qemu/init.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash -exuo pipefail
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Download the installer and ramdisk for intallation
+set -ex
+wget -O installer-vmlinuz http://http.us.debian.org/debian/dists/jessie/main/installer-armhf/current/images/netboot/vmlinuz
+wget -O installer-initrd.gz http://http.us.debian.org/debian/dists/jessie/main/installer-armhf/current/images/netboot/initrd.gz
diff --git a/ci/qemu/initrd_modif/inittab b/ci/qemu/initrd_modif/inittab
new file mode 100644
index 00000000000..064512595fb
--- /dev/null
+++ b/ci/qemu/initrd_modif/inittab
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# /etc/inittab
+# busybox init configuration for debian-installer
+
+# main rc script
+::sysinit:/sbin/reopen-console /sbin/debian-installer-startup
+
+# main setup program
+::respawn:/sbin/reopen-console /sbin/debian-installer
+
+# convenience shells
+tty2::askfirst:-/bin/sh
+tty3::askfirst:-/bin/sh
+
+# logging
+#tty4::respawn:/usr/bin/tail -f /var/log/syslog
+
+# Stuff to do before rebooting
+::ctrlaltdel:/sbin/shutdown > /dev/null 2>&1
+
+# re-exec init on receipt of SIGHUP/SIGUSR1
+::restart:/sbin/init
diff --git a/ci/qemu/install.sh b/ci/qemu/install.sh
new file mode 100755
index 00000000000..8531b033d07
--- /dev/null
+++ b/ci/qemu/install.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+rm -f vda.qcow2
+sudo ./preseed.sh
+qemu-img create -f qcow2 vda.qcow2 10G
+qemu-system-arm -M virt -m 1024 \
+ -kernel installer-vmlinuz \
+ -append BOOT_DEBUG=2,DEBIAN_FRONTEND=noninteractive \
+ -initrd installer-initrd_automated.gz \
+ -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
+ -device virtio-blk-device,drive=hd \
+ -netdev user,id=mynet \
+ -device virtio-net-device,netdev=mynet \
+ -nographic -no-reboot
diff --git a/ci/qemu/mxnet_requirements.txt b/ci/qemu/mxnet_requirements.txt
new file mode 100644
index 00000000000..a2e485efed1
--- /dev/null
+++ b/ci/qemu/mxnet_requirements.txt
@@ -0,0 +1,7 @@
+urllib3<1.23,>=1.21.1
+requests<2.19.0,>=2.18.4
+graphviz<0.9.0,>=0.8.1
+numpy<=1.15.0,>=1.8.2
+mock
+nose
+nose-timer
diff --git a/ci/qemu/preseed.cfg b/ci/qemu/preseed.cfg
new file mode 100644
index 00000000000..23a8fc3baeb
--- /dev/null
+++ b/ci/qemu/preseed.cfg
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+d-i debian-installer/locale string en_US
+d-i keyboard-configuration/xkb-keymap select us
+d-i netcfg/get_hostname string debian-qemu
+d-i netcfg/get_domain string lab
+d-i passwd/root-login boolean true
+d-i passwd/root-password password debian
+d-i passwd/root-password-again password debian
+d-i clock-setup/utc boolean true
+d-i mirror/country string US
+d-i mirror/https/proxy string
+d-i mirror/http/proxy string
+d-i mirror/ftp/proxy string
+d-i mirror/http/countries select US
+d-i mirror/http/hostname string ftp.us.debian.org
+d-i mirror/http/mirror select ftp.us.debian.org
+d-i localechooser/preferred-locale select en_US.UTF-8
+apt-mirror-setup apt-setup/use_mirror boolean false
+apt-mirror-setup apt-setup/mirror/error select Retry
+d-i passwd/username string qemu
+d-i passwd/user-password password qemu
+d-i passwd/user-password-again password qemu
+user-setup-udeb passwd/username string qemu
+user-setup-udeb passwd/user-fullname string qemu
+d-i time/zone string GMT
+d-i partman-auto/choose_recipe select atomic
+#partman-auto partman-auto/select_disk select /var/lib/partman/devices/=dev=vda
+#partman-auto partman-auto/automatically_partition select
+#partman-target partman-target/no_root error
+#partman-auto partman-auto/init_automatically_partition select 50some_device__________regular
+#partman-auto partman-auto/disk string vda
+#partman-auto partman-auto/expert_recipe string \
+# boot-root :: \
+# 100 10000 1000000000 ext4 \
+# $primary{ } \
+# lv_name{ root } \
+# method{ format } \
+# format{ } \
+# use_filesystem{ } \
+# filesystem{ ext4 } \
+# mountpoint{ / } .
+#
+#d-i partman-partitioning/confirm_write_new_label boolean true
+#d-i partman/choose_partition select finish
+#d-i partman/confirm boolean true
+#d-i partman/confirm_nooverwrite boolean true
+#partman-base partman/choose_partition select 90finish__________finish
+#partman-basicfilesystems partman-basicfilesystems/swap_check_failed boolean
+d-i popularity-contest/participate boolean false
+d-i tasksel/first multiselect SSH server, standard system utilities
+d-i debian-installer/main-menu select Finish the installation
+d-i debian-installer/exit/poweroff boolean true
diff --git a/ci/qemu/preseed.sh b/ci/qemu/preseed.sh
new file mode 100755
index 00000000000..ad005548fbb
--- /dev/null
+++ b/ci/qemu/preseed.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash -exuo pipefail
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+rm -rf initrd
+mkdir -p initrd
+cd initrd
+gunzip -c ../installer-initrd.gz | cpio -i
+cp ../preseed.cfg .
+cp ../initrd_modif/inittab etc/inittab
+cp ../initrd_modif/S10syslog lib/debian-installer-startup.d/S10syslog
+find . | cpio --create --format 'newc' | gzip -c > ../installer-initrd_automated.gz
+echo "Done!"
diff --git a/ci/qemu/run.sh b/ci/qemu/run.sh
new file mode 100755
index 00000000000..eeff4e1fdcc
--- /dev/null
+++ b/ci/qemu/run.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash -exuo pipefail
+
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+disk=${1:-vda.qcow2}
+qemu-system-arm -M virt -m 1024 \
+ -kernel vmlinuz-3.16.0-6-armmp-lpae \
+ -initrd initrd.img-3.16.0-6-armmp-lpae \
+ -smp 4 \
+ -append 'root=/dev/vda1' \
+ -drive if=none,file=$disk,format=qcow2,id=hd \
+ -device virtio-blk-device,drive=hd \
+ -netdev user,id=mynet,hostfwd=tcp::2222-:22 \
+ -device virtio-net-device,netdev=mynet \
+ -nographic
+# -display none
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 0eca73fbb02..3ca696b288c 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,3 +1,4 @@
# Requirements for tests, those are installed before running on the virtualenv
mock
nose
+nose-timer
diff --git a/tools/license_header.py b/tools/license_header.py
index f6726891f52..2f12a14795c 100755
--- a/tools/license_header.py
+++ b/tools/license_header.py
@@ -77,7 +77,8 @@
'prepare_mkl.sh',
'example/image-classification/predict-cpp/image-classification-predict.cc',
'src/operator/contrib/ctc_include/',
- 'julia/REQUIRE']
+ 'julia/REQUIRE'
+ ]
# language extensions and the according commment mark
_LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#',
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services