You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nw...@apache.org on 2020/02/19 16:17:41 UTC
[incubator-heron] branch master updated: Java 11 support (#3454)
This is an automated email from the ASF dual-hosted git repository.
nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 1476076 Java 11 support (#3454)
1476076 is described below
commit 14760762b052ed9a6c8aea2988beb5108ae9fc79
Author: se choi <th...@gmail.com>
AuthorDate: Thu Feb 20 01:17:29 2020 +0900
Java 11 support (#3454)
* Support Java 11
* config travis to use oracle jdk 11
* Java 11 support (#3399)
* Support Java 11
* config travis to use oracle jdk 11
* Add check jdk version
* Fix command arguments.
Change insert gc_options
Update list
Fix gc-logging
* Add missing parameter
* typo
* Add pause time
* Add python3 for docker files
* Revert "config travis to use oracle jdk 11"
This reverts commit edaa84f4a6665a1bbd4a93feb67eb67f62476d9a.
* Update unitest
* Update unittest
* Add JDK11 UnitTest
* Add Unittest using Docker.
* Add full test
* Change JDK11
* Ignore PowerMock used for testing is not compatible with Java 11.
* Change JDK11 for travis.yml
* Update python
* Update library requests, pylint
* Revert "Update library requests, pylint"
This reverts commit 7706d34a7b108846b1d4ada8c74e7f6ce65911c0.
* Add ignore
* Refactoring ReflectionUtils
Co-authored-by: Ning Wang <wa...@gmail.com>
---
.travis.yml | 2 +-
WORKSPACE | 13 +-
docker/compile/Dockerfile.centos7 | 10 +-
docker/compile/Dockerfile.debian9 | 11 +-
docker/compile/Dockerfile.ubuntu14.04 | 12 +-
docker/compile/Dockerfile.ubuntu16.04 | 15 +-
docker/compile/Dockerfile.ubuntu18.04 | 5 +-
docker/dist/Dockerfile.dist.centos7 | 6 +-
docker/dist/Dockerfile.dist.debian9 | 4 +-
docker/dist/Dockerfile.dist.ubuntu14.04 | 5 +-
docker/dist/Dockerfile.dist.ubuntu16.04 | 5 +-
docker/dist/Dockerfile.dist.ubuntu18.04 | 4 +-
docker/scripts/test-docker.sh | 64 +++++++
docker/scripts/test-platform.sh | 76 ++++++++
docker/scripts/test-unittest.sh | 175 ++++++++++++++++++
docker/{compile => test}/Dockerfile.centos7 | 11 +-
docker/{compile => test}/Dockerfile.ubuntu18.04 | 7 +-
.../apache/heron/eco/submit/EcoSubmitterTest.java | 2 +
heron/api/src/java/BUILD | 3 +-
.../org/apache/heron/api/HeronSubmitterTest.java | 2 +
heron/common/src/java/BUILD | 1 +
.../apache/heron/common/basics/FileUtilsTest.java | 2 +
.../apache/heron/downloader/DLDownloaderTest.java | 2 +
heron/executor/src/python/heron_executor.py | 129 ++++++--------
.../tests/python/heron_executor_unittest.py | 196 ++++++++++++++++-----
heron/scheduler-core/src/java/BUILD | 1 +
.../apache/heron/scheduler/LaunchRunnerTest.java | 2 +
.../heron/scheduler/RuntimeManagerMainTest.java | 2 +
.../heron/scheduler/RuntimeManagerRunnerTest.java | 2 +
.../apache/heron/scheduler/SchedulerMainTest.java | 2 +
.../apache/heron/scheduler/SubmitterMainTest.java | 2 +
.../heron/scheduler/UpdateTopologyManagerTest.java | 2 +
.../client/HttpServiceSchedulerClientTest.java | 2 +
.../client/SchedulerClientFactoryTest.java | 2 +
.../scheduler/dryrun/SubmitDryRunRenderTest.java | 2 +
.../scheduler/dryrun/UpdateDryRunRenderTest.java | 2 +
.../heron/scheduler/utils/LauncherUtilsTest.java | 2 +
.../heron/scheduler/utils/SchedulerUtilsTest.java | 2 +
.../heron/scheduler/aurora/AuroraLauncherTest.java | 2 +
.../scheduler/aurora/AuroraSchedulerTest.java | 2 +
.../kubernetes/KubernetesLauncherTest.java | 2 +
.../heron/scheduler/local/LocalLauncherTest.java | 2 +
.../scheduler/marathon/MarathonControllerTest.java | 2 +
.../scheduler/marathon/MarathonLauncherTest.java | 2 +
.../heron/scheduler/nomad/NomadSchedulerTest.java | 2 +-
.../heron/scheduler/slurm/SlurmControllerTest.java | 2 +
.../heron/scheduler/slurm/SlurmLauncherTest.java | 2 +
.../heron/scheduler/slurm/SlurmSchedulerTest.java | 2 +
.../scheduler/yarn/HeronExecutorTaskTest.java | 2 +-
.../scheduler/yarn/HeronMasterDriverTest.java | 2 +
heron/simulator/src/java/BUILD | 1 +
.../apache/heron/spi/utils/ReflectionUtils.java | 6 +-
.../apache/heron/spi/common/ConfigLoaderTest.java | 2 +
.../org/apache/heron/spi/common/ConfigTest.java | 3 +-
.../apache/heron/spi/utils/NetworkUtilsTest.java | 5 +-
.../statefulstorage/dlog/DlogStorageTest.java | 2 +
.../statefulstorage/hdfs/HDFSStorageTest.java | 2 +
.../localfs/LocalFileSystemStorageTest.java | 2 +
.../localfs/LocalFileSystemStateManagerTest.java | 2 +
.../heron/statemgr/zookeeper/ZkUtilsTest.java | 2 +
heron/tools/apiserver/src/java/BUILD | 5 +-
61 files changed, 669 insertions(+), 174 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index af9b6eb..7da0cfa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ dist: trusty
language: java
jdk:
- - oraclejdk8
+ - oraclejdk11
addons:
apt:
diff --git a/WORKSPACE b/WORKSPACE
index b8d7bc6..ac4393f 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -258,7 +258,7 @@ maven_jar(
maven_jar(
name = "javax_annotation_javax_annotation_api",
- artifact = "javax.annotation:javax.annotation-api:1.2",
+ artifact = "javax.annotation:javax.annotation-api:1.3.2",
)
maven_jar(
@@ -735,6 +735,17 @@ maven_jar(
artifact = "org.apache.commons:commons-compress:1.14",
)
+maven_jar(
+ name = "javax_xml_bind_jaxb",
+ artifact = "javax.xml.bind:jaxb-api:2.3.0",
+)
+
+maven_jar(
+ name = "javax_activation_activation",
+ artifact = "javax.activation:activation:1.1.1",
+)
+
+
# bookkeeper & distributedlog dependencies
maven_jar(
name = "org_apache_distributedlog_core",
diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos7
index 67a15de..ce36899 100644
--- a/docker/compile/Dockerfile.centos7
+++ b/docker/compile/Dockerfile.centos7
@@ -35,18 +35,18 @@ RUN yum -y install \
libtool \
make \
patch \
- python \
python-devel \
- python-setuptools \
+ python3-devel \
+ python3-setuptools \
zip \
unzip \
wget \
which \
tree \
- java-1.8.0-openjdk \
- java-1.8.0-openjdk-devel
+ java-11-openjdk \
+ java-11-openjdk-devel
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
diff --git a/docker/compile/Dockerfile.debian9 b/docker/compile/Dockerfile.debian9
index 2a3e70a..0af0330 100644
--- a/docker/compile/Dockerfile.debian9
+++ b/docker/compile/Dockerfile.debian9
@@ -30,14 +30,17 @@ RUN apt-get update && apt-get -y install \
git \
libtool \
libtool-bin \
- python \
- python2.7-dev \
+ python-dev \
+ python3-dev \
software-properties-common \
- python-setuptools \
+ python3-setuptools \
tree \
zip \
unzip \
- wget
+ wget \
+ openjdk-11-jdk-headless
+
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
diff --git a/docker/compile/Dockerfile.ubuntu14.04 b/docker/compile/Dockerfile.ubuntu14.04
index 4841616..2d63357 100644
--- a/docker/compile/Dockerfile.ubuntu14.04
+++ b/docker/compile/Dockerfile.ubuntu14.04
@@ -33,18 +33,18 @@ RUN apt-get update && apt-get -y install \
libssl-dev \
git \
libtool \
- python \
- python2.7-dev \
- python-software-properties \
- python-setuptools \
+ python-dev \
+ python3-dev \
+ python3-software-properties \
+ python3-setuptools \
zip \
unzip \
wget \
cmake \
tree \
- openjdk-8-jdk-headless
+ openjdk-11-jdk-headless
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
diff --git a/docker/compile/Dockerfile.ubuntu16.04 b/docker/compile/Dockerfile.ubuntu16.04
index 6a54f47..510f59f 100644
--- a/docker/compile/Dockerfile.ubuntu16.04
+++ b/docker/compile/Dockerfile.ubuntu16.04
@@ -31,19 +31,18 @@ RUN apt-get update && apt-get -y install \
libtool-bin \
libunwind8 \
libunwind-setjmp0-dev \
- python \
- python2.7-dev \
- python-software-properties \
+ python-dev \
+ python3-dev \
+ python3-software-properties \
software-properties-common \
- python-setuptools \
+ python3-setuptools \
tree \
zip \
unzip \
- wget
+ wget \
+ openjdk-11-jdk-headless
-RUN apt-get update && apt-get -y install openjdk-8-jdk-headless
-
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04
index 8b41626..bac758f 100644
--- a/docker/compile/Dockerfile.ubuntu18.04
+++ b/docker/compile/Dockerfile.ubuntu18.04
@@ -29,6 +29,7 @@ RUN apt-get update && apt-get -y install \
libunwind8 \
patch \
python-dev \
+ python3-dev \
wget \
zip \
virtualenv \
@@ -36,9 +37,9 @@ RUN apt-get update && apt-get -y install \
git \
curl \
tree \
- openjdk-8-jdk-headless
+ openjdk-11-jdk-headless
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos7
index d19acc7..d548ff5 100644
--- a/docker/dist/Dockerfile.dist.centos7
+++ b/docker/dist/Dockerfile.dist.centos7
@@ -23,13 +23,13 @@ RUN yum -y install unzip; yum clean all
RUN yum -y install which; yum clean all
RUN yum -y install curl; yum clean all
RUN yum -y install nmap-ncat; yum clean all
-RUN yum -y install python-setuptools; yum clean all
+RUN yum -y install python python3-setuptools; yum clean all
RUN easy_install supervisor
RUN echo_supervisord_conf > /etc/supervisord.conf
-RUN yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel; yum clean all
+RUN yum -y install java-11-openjdk java-11-openjdk-devel; yum clean all
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk
ADD artifacts /heron
diff --git a/docker/dist/Dockerfile.dist.debian9 b/docker/dist/Dockerfile.dist.debian9
index 97953d9..7ba3645 100644
--- a/docker/dist/Dockerfile.dist.debian9
+++ b/docker/dist/Dockerfile.dist.debian9
@@ -20,10 +20,12 @@ FROM openjdk:8-jdk-slim
RUN apt-get -y update && apt-get -y install \
netcat-openbsd \
python \
+ python3 \
unzip \
curl \
vim \
- supervisor && \
+ supervisor \
+ openjdk-11-jdk-headless && \
apt-get clean all && \
rm -rf /var/lib/apt/lists/*
diff --git a/docker/dist/Dockerfile.dist.ubuntu14.04 b/docker/dist/Dockerfile.dist.ubuntu14.04
index a62b9d5..df807aa 100644
--- a/docker/dist/Dockerfile.dist.ubuntu14.04
+++ b/docker/dist/Dockerfile.dist.ubuntu14.04
@@ -19,15 +19,16 @@ FROM ubuntu:14.04
RUN apt-get -y update && apt-get -y install \
python \
+ python3 \
unzip \
software-properties-common \
supervisor \
curl
RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update && \
- apt-get -y install openjdk-8-jdk
+ apt-get -y install openjdk-11-jdk-headless
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN update-ca-certificates -f
ADD artifacts /heron
diff --git a/docker/dist/Dockerfile.dist.ubuntu16.04 b/docker/dist/Dockerfile.dist.ubuntu16.04
index 3a63c18..0fe59f9 100644
--- a/docker/dist/Dockerfile.dist.ubuntu16.04
+++ b/docker/dist/Dockerfile.dist.ubuntu16.04
@@ -21,13 +21,14 @@ RUN apt-get update
RUN apt-get -y install \
python \
+ python3 \
unzip \
software-properties-common \
curl \
supervisor \
- openjdk-8-jdk-headless
+ openjdk-11-jdk-headless
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN update-ca-certificates -f
ADD artifacts /heron
diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04
index 5a21d30..89d4ad7 100644
--- a/docker/dist/Dockerfile.dist.ubuntu18.04
+++ b/docker/dist/Dockerfile.dist.ubuntu18.04
@@ -19,9 +19,9 @@ FROM ubuntu:18.04
RUN apt-get update
RUN apt-get -y install \
- unzip software-properties-common curl supervisor openjdk-8-jdk-headless
+ unzip software-properties-common curl python python3 supervisor openjdk-11-jdk-headless
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN update-ca-certificates -f
ADD artifacts /heron
diff --git a/docker/scripts/test-docker.sh b/docker/scripts/test-docker.sh
new file mode 100755
index 0000000..ca1b4db
--- /dev/null
+++ b/docker/scripts/test-docker.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -o nounset
+set -o errexit
+
+realpath() {
+ echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
+}
+
+DOCKER_DIR=$(dirname $(dirname $(realpath $0)))
+PROJECT_DIR=$(dirname $DOCKER_DIR)
+
+verify_dockerfile_exists() {
+ if [ ! -f $1 ]; then
+ echo "The Dockerfiler $1 does not exist"
+ exit 1
+ fi
+}
+
+dockerfile_path_for_platform() {
+ echo "$SCRATCH_DIR/test/Dockerfile.$1"
+}
+
+copy_bazel_rc_to() {
+ cp $PROJECT_DIR/tools/docker/bazel.rc $1
+}
+
+DOCKER_FILE=$(dockerfile_path_for_platform $TARGET_PLATFORM)
+verify_dockerfile_exists $DOCKER_FILE
+copy_bazel_rc_to $SCRATCH_DIR/bazelrc
+
+echo "Building heron-compiler container"
+docker build -t heron-compiler:$TARGET_PLATFORM -f $DOCKER_FILE $SCRATCH_DIR
+
+echo "Running build in container"
+docker run \
+ --rm \
+ -e TARGET_PLATFORM=$TARGET_PLATFORM \
+ -e SCRATCH_DIR="/scratch" \
+ -e SOURCE_TARBALL="/src.tar.gz" \
+ -e HERON_VERSION=$HERON_VERSION \
+ -e HERON_GIT_REV="${HERON_GIT_REV}" \
+ -e HERON_BUILD_VERSION="${HERON_BUILD_VERSION}" \
+ -e HERON_BUILD_HOST="${HERON_BUILD_HOST}" \
+ -e HERON_BUILD_USER="${HERON_BUILD_USER}" \
+ -e HERON_BUILD_TIME="${HERON_BUILD_TIME}" \
+ -e HERON_TREE_STATUS="${HERON_TREE_STATUS}" \
+ -v "$SOURCE_TARBALL:/src.tar.gz:ro" \
+ -t heron-compiler:$TARGET_PLATFORM /test-platform.sh
diff --git a/docker/scripts/test-platform.sh b/docker/scripts/test-platform.sh
new file mode 100755
index 0000000..c0d7aa1
--- /dev/null
+++ b/docker/scripts/test-platform.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -o nounset
+set -o errexit
+
+# By default bazel creates files with mode 0555 which means they are only able to be read and not written to. This
+# causes cp to fail when trying to overwrite the file. This makes sure that if the file exists we can overwrite it.
+function copyFileToDest() {
+ if [ -f $2 ]; then
+ chmod 755 $2
+ fi
+
+ cp $1 $2
+}
+
+echo "Building heron with version $HERON_VERSION for platform $TARGET_PLATFORM"
+
+mkdir -p $SCRATCH_DIR
+cd $SCRATCH_DIR
+
+echo "Extracting source"
+tar -C . -xzf $SOURCE_TARBALL
+
+if [[ "$TARGET_PLATFORM" =~ "ubuntu" ]]; then
+ CONFIG_PLATFORM=ubuntu
+elif [[ "$TARGET_PLATFORM" =~ "centos" ]]; then
+ CONFIG_PLATFORM=centos
+elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then
+ CONFIG_PLATFORM=darwin
+elif [[ "$TARGET_PLATFORM" =~ "debian" ]]; then
+ CONFIG_PLATFORM=debian
+elif [[ "$TARGET_PLATFORM" =~ "ubuntu_nostyle" ]]; then
+ CONFIG_PLATFORM=ubuntu
+elif [[ "$TARGET_PLATFORM" =~ "centos_nostyle" ]]; then
+ CONFIG_PLATFORM=centos
+elif [[ "$TARGET_PLATFORM" =~ "darwin_nostyle" ]]; then
+ CONFIG_PLATFORM=darwin
+elif [[ "$TARGET_PLATFORM" =~ "debian_nostyle" ]]; then
+ CONFIG_PLATFORM=debian
+else
+ echo "Unknown platform: $TARGET_PLATFORM"
+ exit 1
+fi
+
+bazel version
+./bazel_configure.py
+bazel clean
+
+echo "UnitTest"
+bazel test -c opt --jobs 25 \
+ --config=$CONFIG_PLATFORM \
+ --test_output=all \
+ --test_summary=detailed \
+ heron/... \
+ heronpy/... \
+ examples/... \
+ storm-compatibility-examples/... \
+ eco-storm-examples/... \
+ eco-heron-examples/... \
+ contrib/...
+
diff --git a/docker/scripts/test-unittest.sh b/docker/scripts/test-unittest.sh
new file mode 100755
index 0000000..0192a09
--- /dev/null
+++ b/docker/scripts/test-unittest.sh
@@ -0,0 +1,175 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -o errexit
+
+realpath() {
+ echo "$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
+}
+
+DOCKER_DIR=$(dirname $(dirname $(realpath $0)))
+PROJECT_DIR=$(dirname $DOCKER_DIR)
+SCRATCH_DIR="$HOME/.heron-compile"
+SRC_TAR="$SCRATCH_DIR/src.tar.gz"
+
+heron_git_release() {
+ local git_release=$(git rev-parse --abbrev-ref HEAD)
+ if [[ $? != 0 ]];
+ then
+ exit 1
+ fi
+ if [ "${git_release}" = "HEAD" ];
+ then
+ git_release=$(git describe --tags)
+ if [[ $? != 0 ]];
+ then
+ exit 1
+ fi
+ fi
+ echo $git_release
+}
+
+heron_git_rev() {
+ local git_rev=$(git rev-parse HEAD)
+ if [[ $? != 0 ]];
+ then
+ exit 1
+ fi
+ echo $git_rev
+}
+
+heron_build_host() {
+ local build_host=$(hostname)
+ echo $build_host
+}
+
+heron_build_user() {
+ local build_user=$USER
+ echo $build_user
+}
+
+heron_build_time() {
+ local build_time=$(LC_ALL=en_EN.utf8 date)
+ echo $build_time
+}
+
+heron_build_timestamp() {
+ local build_timestamp=$(date +%s000)
+ echo $build_timestamp
+}
+
+heron_tree_status() {
+ local tree_status=""
+ git diff-index --quiet HEAD --
+ if [[ $? == 0 ]];
+ then
+ tree_status="Clean"
+ else
+ tree_status="Modified"
+ fi
+ echo $tree_status
+}
+
+cleanup() {
+ if [ -f $SRC_TAR ]; then
+ echo "Cleaning up scratch dir"
+ rm -rf $SCRATCH_DIR
+ fi
+}
+
+trap cleanup EXIT
+
+generate_source() {
+ echo "Generating source tarball"
+ tar --exclude-from=$DOCKER_DIR/.tarignore -C $PROJECT_DIR -czf $SRC_TAR .
+}
+
+verify_source_exists() {
+ if [ ! -f $1 ]; then
+ echo "The source provided $1 does not exist"
+ exit 1
+ fi
+}
+
+setup_scratch_dir() {
+ mkdir -p $1/artifacts
+ cp -r $DOCKER_DIR/* $1
+}
+
+setup_output_dir() {
+ echo "Creating output directory $1"
+ mkdir -p $1
+}
+
+run_build() {
+ TARGET_PLATFORM=$1
+ HERON_VERSION=$2
+ SOURCE_TARBALL=$3
+
+ setup_scratch_dir $SCRATCH_DIR
+
+ if [ -z "$SOURCE_TARBALL" ]; then
+ generate_source
+ SOURCE_TARBALL=$SRC_TAR
+ else
+ SOURCE_TARBALL=$(realpath $3)
+ fi
+ verify_source_exists $SOURCE_TARBALL
+
+ export TARGET_PLATFORM=${TARGET_PLATFORM}
+ export HERON_VERSION=${HERON_VERSION}
+ export SCRATCH_DIR=${SCRATCH_DIR}
+ export SOURCE_TARBALL=${SOURCE_TARBALL}
+
+ export HERON_BUILD_VERSION="${HERON_BUILD_VERSION:-$(heron_git_release)}"
+ export HERON_GIT_REV="${HERON_GIT_REV:-$(heron_git_rev)}"
+ export HERON_BUILD_HOST="${HERON_GIT_HOST:-$(heron_build_host)}"
+ export HERON_BUILD_USER="${HERON_BUILD_USER:-$(heron_build_user)}"
+ export HERON_BUILD_TIME="${HERON_BUILD_TIME:-$(heron_build_time)}"
+ export HERON_TREE_STATUS="${HERON_TREE_STATUS:-$(heron_tree_status)}"
+
+ if [ $TARGET_PLATFORM = "darwin" ]; then
+ docker/scripts/test-platform.sh
+ else
+ docker/scripts/test-docker.sh
+ fi
+}
+
+case $# in
+ 2)
+ run_build $1 $2 ""
+ ;;
+
+ 3)
+ run_build $1 $2 $3
+ ;;
+
+ *)
+ echo "Usage: $0 <platform> <version_string> [source-tarball]"
+ echo " "
+ echo "Script to test heron artifacts for different platforms"
+ echo " "
+ echo "Platforms Supported: darwin, ubuntu18.04, centos7"
+ echo " "
+ echo "Example:"
+ echo " ./test-unittest.sh centos7 0.20.2"
+ echo " "
+ echo "NOTE: If running on OSX, the output directory will need to "
+ echo " be under /Users so virtualbox has access to."
+ exit 1
+ ;;
+esac
diff --git a/docker/compile/Dockerfile.centos7 b/docker/test/Dockerfile.centos7
similarity index 88%
copy from docker/compile/Dockerfile.centos7
copy to docker/test/Dockerfile.centos7
index 67a15de..8122b73 100644
--- a/docker/compile/Dockerfile.centos7
+++ b/docker/test/Dockerfile.centos7
@@ -35,22 +35,21 @@ RUN yum -y install \
libtool \
make \
patch \
- python \
python-devel \
- python-setuptools \
+ python3-devel \
+ python3-setuptools \
zip \
unzip \
wget \
which \
tree \
- java-1.8.0-openjdk \
- java-1.8.0-openjdk-devel
+ java-11-openjdk-devel
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
&& /tmp/bazel.sh
ADD bazelrc /root/.bazelrc
-ADD scripts/compile-platform.sh /compile-platform.sh
+ADD scripts/test-platform.sh /test-platform.sh
diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/test/Dockerfile.ubuntu18.04
similarity index 90%
copy from docker/compile/Dockerfile.ubuntu18.04
copy to docker/test/Dockerfile.ubuntu18.04
index 8b41626..e9c2e82 100644
--- a/docker/compile/Dockerfile.ubuntu18.04
+++ b/docker/test/Dockerfile.ubuntu18.04
@@ -29,6 +29,7 @@ RUN apt-get update && apt-get -y install \
libunwind8 \
patch \
python-dev \
+ python3-dev \
wget \
zip \
virtualenv \
@@ -36,13 +37,13 @@ RUN apt-get update && apt-get -y install \
git \
curl \
tree \
- openjdk-8-jdk-headless
+ openjdk-11-jdk-headless
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \
&& chmod +x /tmp/bazel.sh \
&& /tmp/bazel.sh
ADD bazelrc /root/.bazelrc
-ADD scripts/compile-platform.sh /compile-platform.sh
+ADD scripts/test-platform.sh /test-platform.sh
diff --git a/eco/tests/java/org/apache/heron/eco/submit/EcoSubmitterTest.java b/eco/tests/java/org/apache/heron/eco/submit/EcoSubmitterTest.java
index c386f60..51a7c27 100644
--- a/eco/tests/java/org/apache/heron/eco/submit/EcoSubmitterTest.java
+++ b/eco/tests/java/org/apache/heron/eco/submit/EcoSubmitterTest.java
@@ -23,6 +23,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -37,6 +38,7 @@ import static org.mockito.Matchers.anyString;
import static org.mockito.internal.verification.VerificationModeFactory.times;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({StormSubmitter.class, HeronSubmitter.class})
public class EcoSubmitterTest {
diff --git a/heron/api/src/java/BUILD b/heron/api/src/java/BUILD
index 46d389c..dec342f 100644
--- a/heron/api/src/java/BUILD
+++ b/heron/api/src/java/BUILD
@@ -16,6 +16,7 @@ api_deps_files = \
heron_java_api_proto_files() + [
":classification",
"//heron/common/src/java:basics-java",
+ "@javax_xml_bind_jaxb//jar",
]
# Low Level Api
@@ -46,7 +47,7 @@ java_library(
srcs = glob(["org/apache/heron/api/**/*.java", "org/apache/heron/streamlet/**/*.java"]),
deps = api_deps_files + [
"//third_party/java:kryo-neverlink",
- "@org_apache_commons_commons_lang3//jar"
+ "@org_apache_commons_commons_lang3//jar",
]
)
diff --git a/heron/api/tests/java/org/apache/heron/api/HeronSubmitterTest.java b/heron/api/tests/java/org/apache/heron/api/HeronSubmitterTest.java
index 26295d9..f240168 100644
--- a/heron/api/tests/java/org/apache/heron/api/HeronSubmitterTest.java
+++ b/heron/api/tests/java/org/apache/heron/api/HeronSubmitterTest.java
@@ -28,6 +28,7 @@ import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -49,6 +50,7 @@ import org.apache.heron.common.basics.ByteAmount;
* This class covers HeronSubmitter Unit Tests for both positive and negative cases
*/
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
public class HeronSubmitterTest {
@Rule
diff --git a/heron/common/src/java/BUILD b/heron/common/src/java/BUILD
index 259abf3..5677305 100644
--- a/heron/common/src/java/BUILD
+++ b/heron/common/src/java/BUILD
@@ -20,6 +20,7 @@ java_library(
srcs = glob(["**/config/*.java"]),
deps = [
":basics-java",
+ "@javax_xml_bind_jaxb//jar",
"@org_yaml_snakeyaml//jar",
],
)
diff --git a/heron/common/tests/java/org/apache/heron/common/basics/FileUtilsTest.java b/heron/common/tests/java/org/apache/heron/common/basics/FileUtilsTest.java
index b732581..3d898f4 100644
--- a/heron/common/tests/java/org/apache/heron/common/basics/FileUtilsTest.java
+++ b/heron/common/tests/java/org/apache/heron/common/basics/FileUtilsTest.java
@@ -30,6 +30,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Matchers;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -39,6 +40,7 @@ import static org.mockito.Mockito.times;
* FileUtils Tester.
*/
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(FileUtils.class)
public class FileUtilsTest {
diff --git a/heron/downloaders/tests/java/org/apache/heron/downloader/DLDownloaderTest.java b/heron/downloaders/tests/java/org/apache/heron/downloader/DLDownloaderTest.java
index 6f9a68d..15caf0b 100644
--- a/heron/downloaders/tests/java/org/apache/heron/downloader/DLDownloaderTest.java
+++ b/heron/downloaders/tests/java/org/apache/heron/downloader/DLDownloaderTest.java
@@ -28,6 +28,7 @@ import java.nio.file.Paths;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -52,6 +53,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(Extractor.class)
public class DLDownloaderTest {
diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py
index 934d69d..208cf77 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -37,6 +37,7 @@ import time
import yaml
import socket
import traceback
+import itertools
from heron.common.src.python.utils import log
from heron.common.src.python.utils import proc
@@ -431,21 +432,6 @@ class HeronExecutor(object):
# for instance, the default -Xmx in Twitter mesos machine is around 18GB
'-Xmx1024M',
'-XX:+PrintCommandLineFlags',
- '-verbosegc',
- '-XX:+PrintGCDetails',
- '-XX:+PrintGCTimeStamps',
- '-XX:+PrintGCDateStamps',
- '-XX:+PrintGCCause',
- '-XX:+UseGCLogFileRotation',
- '-XX:NumberOfGCLogFiles=5',
- '-XX:GCLogFileSize=100M',
- '-XX:+PrintPromotionFailure',
- '-XX:+PrintTenuringDistribution',
- '-XX:+PrintHeapAtGC',
- '-XX:+HeapDumpOnOutOfMemoryError',
- '-XX:+UseConcMarkSweepGC',
- '-XX:+PrintCommandLineFlags',
- '-Xloggc:log-files/gc.' + metricsManagerId + '.log',
'-Djava.net.preferIPv4Stack=true',
'-cp',
self.metrics_manager_classpath,
@@ -461,6 +447,8 @@ class HeronExecutor(object):
'--override-config-file=' + self.override_config_file,
'--sink-config-file=' + sink_config_file]
+ # Insert GC Options
+ metricsmgr_cmd = self._get_java_gc_instance_cmd(metricsmgr_cmd, metricsManagerId)
return Command(metricsmgr_cmd, self.shell_env)
def _get_metrics_cache_cmd(self):
@@ -472,21 +460,6 @@ class HeronExecutor(object):
# for instance, the default -Xmx in Twitter mesos machine is around 18GB
'-Xmx1024M',
'-XX:+PrintCommandLineFlags',
- '-verbosegc',
- '-XX:+PrintGCDetails',
- '-XX:+PrintGCTimeStamps',
- '-XX:+PrintGCDateStamps',
- '-XX:+PrintGCCause',
- '-XX:+UseGCLogFileRotation',
- '-XX:NumberOfGCLogFiles=5',
- '-XX:GCLogFileSize=100M',
- '-XX:+PrintPromotionFailure',
- '-XX:+PrintTenuringDistribution',
- '-XX:+PrintHeapAtGC',
- '-XX:+HeapDumpOnOutOfMemoryError',
- '-XX:+UseConcMarkSweepGC',
- '-XX:+PrintCommandLineFlags',
- '-Xloggc:log-files/gc.metricscache.log',
'-Djava.net.preferIPv4Stack=true',
'-cp',
self.metricscache_manager_classpath,
@@ -503,6 +476,8 @@ class HeronExecutor(object):
"--role", self.role,
"--environment", self.environment]
+ # Insert GC Options
+ metricscachemgr_cmd = self._get_java_gc_instance_cmd(metricscachemgr_cmd, 'metricscache')
return Command(metricscachemgr_cmd, self.shell_env)
def _get_healthmgr_cmd(self):
@@ -514,21 +489,6 @@ class HeronExecutor(object):
# for instance, the default -Xmx in Twitter mesos machine is around 18GB
'-Xmx1024M',
'-XX:+PrintCommandLineFlags',
- '-verbosegc',
- '-XX:+PrintGCDetails',
- '-XX:+PrintGCTimeStamps',
- '-XX:+PrintGCDateStamps',
- '-XX:+PrintGCCause',
- '-XX:+UseGCLogFileRotation',
- '-XX:NumberOfGCLogFiles=5',
- '-XX:GCLogFileSize=100M',
- '-XX:+PrintPromotionFailure',
- '-XX:+PrintTenuringDistribution',
- '-XX:+PrintHeapAtGC',
- '-XX:+HeapDumpOnOutOfMemoryError',
- '-XX:+UseConcMarkSweepGC',
- '-XX:+PrintCommandLineFlags',
- '-Xloggc:log-files/gc.healthmgr.log',
'-Djava.net.preferIPv4Stack=true',
'-cp', self.health_manager_classpath,
healthmgr_main_class,
@@ -538,6 +498,8 @@ class HeronExecutor(object):
"--topology_name", self.topology_name,
"--metricsmgr_port", self.metrics_manager_port]
+ # Insert GC Options
+ healthmgr_cmd = self._get_java_gc_instance_cmd(healthmgr_cmd, 'healthmgr')
return Command(healthmgr_cmd, self.shell_env)
def _get_tmaster_processes(self):
@@ -618,6 +580,46 @@ class HeronExecutor(object):
def _get_jvm_instance_cmd(self):
return Command(os.path.join(self.heron_java_home, 'bin/java'), self.shell_env)
+ def _get_java_major_version(self):
+ return int(self._get_jvm_version().split(".")[0])
+
+ def _get_java_gc_instance_cmd(self, cmd, gc_name):
+ gc_cmd = ['-verbosegc']
+ if self._get_java_major_version() >= 9:
+ gc_cmd += [
+ '-XX:+UseG1GC',
+ '-XX:+ParallelRefProcEnabled',
+ '-XX:+UseStringDeduplication',
+ '-XX:MaxGCPauseMillis=100',
+ '-XX:InitiatingHeapOccupancyPercent=30',
+ '-XX:+HeapDumpOnOutOfMemoryError',
+ '-XX:ParallelGCThreads=4',
+ '-Xlog:gc*,safepoint=info:file=' + self.log_dir + '/gc.' + gc_name +
+ '.log:tags,time,uptime,level:filecount=5,filesize=100M']
+ else:
+ gc_cmd += [
+ '-XX:+UseConcMarkSweepGC',
+ '-XX:+CMSScavengeBeforeRemark',
+ '-XX:TargetSurvivorRatio=90',
+ '-XX:+PrintGCDetails',
+ '-XX:+PrintGCTimeStamps',
+ '-XX:+PrintGCDateStamps',
+ '-XX:+PrintGCCause',
+ '-XX:+UseGCLogFileRotation',
+ '-XX:NumberOfGCLogFiles=5',
+ '-XX:GCLogFileSize=100M',
+ '-XX:+PrintPromotionFailure',
+ '-XX:+PrintTenuringDistribution',
+ '-XX:+PrintHeapAtGC',
+ '-XX:+HeapDumpOnOutOfMemoryError',
+ '-XX:ParallelGCThreads=4',
+ '-Xloggc:' + self.log_dir + '/gc.' + gc_name + '.log']
+ try:
+ cp_index = cmd.index('-cp')
+ return list(itertools.chain(*[cmd[0:cp_index], gc_cmd, cmd[cp_index:]]))
+ except ValueError:
+ return cmd
+
def _get_jvm_instance_options(self, instance_id, component_name, remote_debugger_port):
code_cache_size_mb = 64
java_metasize_mb = 128
@@ -644,28 +646,13 @@ class HeronExecutor(object):
'-XX:Max%s=%dM' % (java_metasize_param, java_metasize_mb),
'-XX:%s=%dM' % (java_metasize_param, java_metasize_mb),
'-XX:ReservedCodeCacheSize=%dM' % code_cache_size_mb,
- '-XX:+CMSScavengeBeforeRemark',
- '-XX:TargetSurvivorRatio=90',
'-XX:+PrintCommandLineFlags',
- '-verbosegc',
- '-XX:+PrintGCDetails',
- '-XX:+PrintGCTimeStamps',
- '-XX:+PrintGCDateStamps',
- '-XX:+PrintGCCause',
- '-XX:+UseGCLogFileRotation',
- '-XX:NumberOfGCLogFiles=5',
- '-XX:GCLogFileSize=100M',
- '-XX:+PrintPromotionFailure',
- '-XX:+PrintTenuringDistribution',
- '-XX:+PrintHeapAtGC',
- '-XX:+HeapDumpOnOutOfMemoryError',
- '-XX:+UseConcMarkSweepGC',
- '-XX:ParallelGCThreads=4',
- '-Xloggc:log-files/gc.%s.log' % instance_id,
'-Djava.net.preferIPv4Stack=true',
'-cp',
'%s:%s'% (self.instance_classpath, self.classpath)]
+ # Insert GC Options
+ instance_options = self._get_java_gc_instance_cmd(instance_options, instance_id)
# Append debugger ports when it is available
if remote_debugger_port:
instance_options.append('-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=%s'
@@ -849,21 +836,6 @@ class HeronExecutor(object):
'-Xms%dM' % ckptmgr_ram_mb,
'-Xmx%dM' % ckptmgr_ram_mb,
'-XX:+PrintCommandLineFlags',
- '-verbosegc',
- '-XX:+PrintGCDetails',
- '-XX:+PrintGCTimeStamps',
- '-XX:+PrintGCDateStamps',
- '-XX:+PrintGCCause',
- '-XX:+UseGCLogFileRotation',
- '-XX:NumberOfGCLogFiles=5',
- '-XX:GCLogFileSize=100M',
- '-XX:+PrintPromotionFailure',
- '-XX:+PrintTenuringDistribution',
- '-XX:+PrintHeapAtGC',
- '-XX:+HeapDumpOnOutOfMemoryError',
- '-XX:+UseConcMarkSweepGC',
- '-XX:+UseConcMarkSweepGC',
- '-Xloggc:log-files/gc.' + ckptmgr_id + '.log',
'-Djava.net.preferIPv4Stack=true',
'-cp',
self.checkpoint_manager_classpath,
@@ -875,6 +847,9 @@ class HeronExecutor(object):
'-f' + self.stateful_config_file,
'-o' + self.override_config_file,
'-g' + self.heron_internals_config_file]
+
+ # Insert GC Options
+ ckptmgr_cmd = self._get_java_gc_instance_cmd(ckptmgr_cmd, ckptmgr_id)
retval = {}
retval[self.ckptmgr_ids[self.shard]] = Command(ckptmgr_cmd, self.shell_env)
diff --git a/heron/executor/tests/python/heron_executor_unittest.py b/heron/executor/tests/python/heron_executor_unittest.py
index 1e9303a..061e054 100644
--- a/heron/executor/tests/python/heron_executor_unittest.py
+++ b/heron/executor/tests/python/heron_executor_unittest.py
@@ -72,7 +72,7 @@ class MockExecutor(HeronExecutor):
# pylint: disable=no-self-use
def _load_logging_dir(self, heron_internals_config_file):
- return "fake_dir"
+ return "log-files"
def _run_process(self, name, cmd, env=None):
popen = MockPOpen()
@@ -82,12 +82,13 @@ class MockExecutor(HeronExecutor):
def _get_jvm_version(self):
return "1.8.y.x"
+
class HeronExecutorTest(unittest.TestCase):
"""Unittest for Heron Executor"""
def get_expected_shell_command(container_id):
return 'heron_shell_binary --port=shell-port ' \
- '--log_file_prefix=fake_dir/heron-shell-%s.log ' \
+ '--log_file_prefix=log-files/heron-shell-%s.log ' \
'--secret=topid' % container_id
def build_packing_plan(self, instance_distribution):
@@ -104,57 +105,68 @@ class HeronExecutorTest(unittest.TestCase):
# pylint: disable=no-self-argument
def get_expected_metricsmgr_command(container_id):
- return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
- "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
- "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
- "-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
- "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
- "-Xloggc:log-files/gc.metricsmgr-%d.log -Djava.net.preferIPv4Stack=true " \
+ return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
+ "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
+ "-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
+ "-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
+ "-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ "-Xloggc:log-files/gc.metricsmgr-%d.log " \
"-cp metricsmgr_classpath org.apache.heron.metricsmgr.MetricsManager " \
"--id=metricsmgr-%d --port=metricsmgr_port " \
- "--topology=topname --cluster=cluster --role=role --environment=environ --topology-id=topid " \
- "--system-config-file=%s --override-config-file=%s --sink-config-file=metrics_sinks_config_file" %\
+ "--topology=topname --cluster=cluster --role=role --environment=environ " \
+ "--topology-id=topid " \
+ "--system-config-file=%s --override-config-file=%s " \
+ "--sink-config-file=metrics_sinks_config_file" %\
(container_id, container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
def get_expected_metricscachemgr_command():
- return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
- "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
- "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
- "-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
- "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
- "-Xloggc:log-files/gc.metricscache.log -Djava.net.preferIPv4Stack=true " \
- "-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
- "--metricscache_id metricscache-0 --master_port metricscachemgr_masterport " \
- "--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
- "--system_config_file %s --override_config_file %s " \
- "--sink_config_file metrics_sinks_config_file " \
- "--cluster cluster --role role --environment environ" %\
- (INTERNAL_CONF_PATH, OVERRIDE_PATH)
+ return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
+ "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
+ "-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
+ "-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
+ "-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ "-Xloggc:log-files/gc.metricscache.log " \
+ "-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
+ "--metricscache_id metricscache-0 --master_port metricscachemgr_masterport " \
+ "--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
+ "--system_config_file %s --override_config_file %s " \
+ "--sink_config_file metrics_sinks_config_file " \
+ "--cluster cluster --role role --environment environ" %\
+ (INTERNAL_CONF_PATH, OVERRIDE_PATH)
def get_expected_healthmgr_command():
- return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags -verbosegc " \
- "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
- "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
- "-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
- "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+PrintCommandLineFlags " \
- "-Xloggc:log-files/gc.healthmgr.log -Djava.net.preferIPv4Stack=true " \
- "-cp scheduler_classpath:healthmgr_classpath " \
- "org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
- "--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"
+ return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
+ "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
+ "-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
+ "-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
+ "-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ "-Xloggc:log-files/gc.healthmgr.log " \
+ "-cp scheduler_classpath:healthmgr_classpath " \
+ "org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
+ "--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"
def get_expected_instance_command(component_name, instance_id, container_id):
instance_name = "container_%d_%s_%d" % (container_id, component_name, instance_id)
return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxMetaspaceSize=128M " \
- "-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+CMSScavengeBeforeRemark " \
- "-XX:TargetSurvivorRatio=90 -XX:+PrintCommandLineFlags -verbosegc -XX:+PrintGCDetails " \
- "-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+PrintGCCause " \
- "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M " \
- "-XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC " \
- "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:ParallelGCThreads=4 " \
- "-Xloggc:log-files/gc.%s.log -Djava.net.preferIPv4Stack=true " \
+ "-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseConcMarkSweepGC -XX:+CMSScavengeBeforeRemark -XX:TargetSurvivorRatio=90 " \
+ "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps " \
+ "-XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 " \
+ "-XX:GCLogFileSize=100M -XX:+PrintPromotionFailure -XX:+PrintTenuringDistribution " \
+ "-XX:+PrintHeapAtGC -XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ "-Xloggc:log-files/gc.%s.log " \
"-cp instance_classpath:classpath -XX:+HeapDumpOnOutOfMemoryError " \
- "org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid -instance_id %s -component_name %s -task_id %d -component_index 0 -stmgr_id stmgr-%d " \
- "-stmgr_port tmaster_controller_port -metricsmgr_port metricsmgr_port -system_config_file %s -override_config_file %s" \
+ "org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid " \
+ "-instance_id %s -component_name %s -task_id %d -component_index 0 -stmgr_id stmgr-%d " \
+ "-stmgr_port tmaster_controller_port -metricsmgr_port metricsmgr_port " \
+ "-system_config_file %s -override_config_file %s" \
% (instance_name, instance_name, component_name, instance_id,
container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
@@ -289,7 +301,6 @@ class HeronExecutorTest(unittest.TestCase):
command = "./heron-executor %s" % (" ".join(args))
return command.split()
-
def test_update_packing_plan(self):
self.executor_0.update_packing_plan(self.packing_plan_expected)
@@ -373,3 +384,104 @@ class HeronExecutorTest(unittest.TestCase):
self.assertEqual(expected_process.name, found_processes[pid].name)
self.assertEqual(expected_process.command, found_processes[pid].command_str)
self.assertEqual(1, found_processes[pid].attempts)
+
+
+class MockExecutorJDK11(HeronExecutor):
+ """
+ mock executor that overrides methods that don't apply to unit tests, like running processes
+ """
+ def __init__(self, args):
+ self.processes = []
+ super(MockExecutorJDK11, self).__init__(args, None)
+
+ # pylint: disable=no-self-use
+ def _load_logging_dir(self, heron_internals_config_file):
+ return "log-files"
+
+ def _run_process(self, name, cmd, env=None):
+ popen = MockPOpen()
+ self.processes.append(ProcessInfo(popen, name, cmd))
+ return popen
+
+ def _get_jvm_version(self):
+ return "11.0.6"
+
+
+class HeronExecutorJDK11Test(unittest.TestCase):
+ """Unittest for Heron Executor"""
+
+ def __init__(self, args):
+ super(HeronExecutorJDK11Test, self).__init__(args, None)
+
+ # pylint: disable=no-self-argument
+ def get_expected_metricsmgr_command(container_id):
+ return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
+ "-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
+ "-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ "-Xlog:gc*,safepoint=info:file=log-files/gc.metricsmgr-%d.log:tags,time,uptime," \
+ "level:filecount=5,filesize=100M " \
+ "-cp metricsmgr_classpath org.apache.heron.metricsmgr.MetricsManager " \
+ "--id=metricsmgr-%d --port=metricsmgr_port " \
+ "--topology=topname --cluster=cluster --role=role --environment=environ " \
+ "--topology-id=topid " \
+ "--system-config-file=%s --override-config-file=%s " \
+ "--sink-config-file=metrics_sinks_config_file" % \
+ (container_id, container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
+
+ def get_expected_metricscachemgr_command():
+ return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
+ "-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
+ "-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ " -Xlog:gc*,safepoint=info:file=log-files/gc.metricscache.log:tags,time,uptime," \
+ "level:filecount=5,filesize=100M " \
+ "-cp metricscachemgr_classpath org.apache.heron.metricscachemgr.MetricsCacheManager " \
+ "--metricscache_id metricscache-0 --master_port metricscachemgr_masterport " \
+ "--stats_port metricscachemgr_statsport --topology_name topname --topology_id topid " \
+ "--system_config_file %s --override_config_file %s " \
+ "--sink_config_file metrics_sinks_config_file " \
+ "--cluster cluster --role role --environment environ" % \
+ (INTERNAL_CONF_PATH, OVERRIDE_PATH)
+
+ def get_expected_healthmgr_command():
+ return "heron_java_home/bin/java -Xmx1024M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
+ "-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
+ "-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ " -Xlog:gc*,safepoint=info:file=log-files/gc.healthmgr.log:tags,time,uptime," \
+ "level:filecount=5,filesize=100M " \
+ "-cp scheduler_classpath:healthmgr_classpath " \
+ "org.apache.heron.healthmgr.HealthManager --cluster cluster --role role " \
+ "--environment environ --topology_name topname --metricsmgr_port metricsmgr_port"
+
+ def get_expected_instance_command(component_name, instance_id, container_id):
+ instance_name = "container_%d_%s_%d" % (container_id, component_name, instance_id)
+ return "heron_java_home/bin/java -Xmx320M -Xms320M -Xmn160M -XX:MaxMetaspaceSize=128M " \
+ "-XX:MetaspaceSize=128M -XX:ReservedCodeCacheSize=64M -XX:+PrintCommandLineFlags " \
+ "-Djava.net.preferIPv4Stack=true -verbosegc " \
+ "-XX:+UseG1GC -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication " \
+ "-XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=30 " \
+ "-XX:+HeapDumpOnOutOfMemoryError -XX:ParallelGCThreads=4 " \
+ "-Xloggc:log-files/gc.%s.log " \
+ "-cp instance_classpath:classpath -XX:+HeapDumpOnOutOfMemoryError " \
+ "org.apache.heron.instance.HeronInstance -topology_name topname -topology_id topid " \
+ "-instance_id %s -component_name %s -task_id %d -component_index 0 -stmgr_id stmgr-%d " \
+ "-stmgr_port tmaster_controller_port -metricsmgr_port metricsmgr_port " \
+ "-system_config_file %s -override_config_file %s" \
+ % (instance_name, instance_name, component_name, instance_id,
+ container_id, INTERNAL_CONF_PATH, OVERRIDE_PATH)
+
+ def setUp(self):
+ MockPOpen.set_next_pid(37)
+ self.maxDiff = None
+ self.executor_0 = MockExecutorJDK11(self.get_args(0))
+ self.executor_1 = MockExecutorJDK11(self.get_args(1))
+ self.executor_7 = MockExecutorJDK11(self.get_args(7))
+ self.packing_plan_expected = self.build_packing_plan({
+ 1:[('word', '3', '0'), ('exclaim1', '2', '0'), ('exclaim1', '1', '0')],
+ 7:[('word', '11', '0'), ('exclaim1', '210', '0')],
+ })
diff --git a/heron/scheduler-core/src/java/BUILD b/heron/scheduler-core/src/java/BUILD
index b471af4..275ed3b 100644
--- a/heron/scheduler-core/src/java/BUILD
+++ b/heron/scheduler-core/src/java/BUILD
@@ -7,6 +7,7 @@ common_deps_files = [
"//heron/api/src/java:classification",
"@commons_cli_commons_cli//jar",
"@com_google_guava_guava//jar",
+ "@javax_xml_bind_jaxb//jar",
"@org_apache_commons_commons_lang3//jar",
"//third_party/java:jackson",
]
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/LaunchRunnerTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/LaunchRunnerTest.java
index 5b498da..2257c28 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/LaunchRunnerTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/LaunchRunnerTest.java
@@ -27,6 +27,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -69,6 +70,7 @@ import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(LauncherUtils.class)
public class LaunchRunnerTest {
private static final String TOPOLOGY_NAME = "testTopology";
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerMainTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerMainTest.java
index 086e4fc..c8827f2 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerMainTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerMainTest.java
@@ -25,6 +25,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -56,6 +57,7 @@ import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"jdk.internal.reflect.*", "jdk.internal.loader.*"})
public class RuntimeManagerMainTest {
private static final String TOPOLOGY_NAME = "topologyName";
private static final String TOPOLOGY_ID = "topologyId";
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerRunnerTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerRunnerTest.java
index 88a3a2c..eb3ea3e 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerRunnerTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/RuntimeManagerRunnerTest.java
@@ -27,6 +27,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -57,6 +58,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
public class RuntimeManagerRunnerTest {
private static final String TOPOLOGY_NAME = "testTopology";
private final Config config = mock(Config.class);
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SchedulerMainTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SchedulerMainTest.java
index b0af302..249ca62 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SchedulerMainTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SchedulerMainTest.java
@@ -30,6 +30,7 @@ import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -68,6 +69,7 @@ import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({
TopologyUtils.class, ReflectionUtils.class, SchedulerUtils.class, TopologyAPI.Topology.class})
public class SchedulerMainTest {
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SubmitterMainTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SubmitterMainTest.java
index aa59c89..a3ebeca 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SubmitterMainTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/SubmitterMainTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -59,6 +60,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({LauncherUtils.class, ReflectionUtils.class})
public class SubmitterMainTest {
private static final String TOPOLOGY_NAME = "topologyName";
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/UpdateTopologyManagerTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/UpdateTopologyManagerTest.java
index 0601662..17f70e5 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/UpdateTopologyManagerTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/UpdateTopologyManagerTest.java
@@ -32,6 +32,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -66,6 +67,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
public class UpdateTopologyManagerTest {
private static final String TOPOLOGY_NAME = "topologyName";
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/HttpServiceSchedulerClientTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/HttpServiceSchedulerClientTest.java
index 877a033..47c9122 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/HttpServiceSchedulerClientTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/HttpServiceSchedulerClientTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -36,6 +37,7 @@ import org.apache.heron.spi.common.Config;
import org.apache.heron.spi.utils.NetworkUtils;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(NetworkUtils.class)
public class HttpServiceSchedulerClientTest {
private static final String TOPOLOGY_NAME = "topologyName";
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/SchedulerClientFactoryTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/SchedulerClientFactoryTest.java
index 6941295..2ba18f9 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/SchedulerClientFactoryTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/client/SchedulerClientFactoryTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -36,6 +37,7 @@ import org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor;
import org.apache.heron.spi.utils.ReflectionUtils;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
public class SchedulerClientFactoryTest {
private static final String TOPOLOGY_NAME = "shiwei_0924_jiayou";
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/SubmitDryRunRenderTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/SubmitDryRunRenderTest.java
index 316ca4d..9b2f9df 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/SubmitDryRunRenderTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/SubmitDryRunRenderTest.java
@@ -29,6 +29,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -44,6 +45,7 @@ import static org.apache.heron.spi.packing.PackingPlan.ContainerPlan;
import static org.junit.Assert.assertEquals;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(TopologyAPI.Topology.class)
public class SubmitDryRunRenderTest {
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/UpdateDryRunRenderTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/UpdateDryRunRenderTest.java
index 3514bfd..b7e2877 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/UpdateDryRunRenderTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/dryrun/UpdateDryRunRenderTest.java
@@ -30,6 +30,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -45,6 +46,7 @@ import static org.apache.heron.spi.packing.PackingPlan.ContainerPlan;
import static org.junit.Assert.assertEquals;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(TopologyAPI.Topology.class)
public class UpdateDryRunRenderTest {
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/LauncherUtilsTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/LauncherUtilsTest.java
index 3ba0f7c..f05ab96 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/LauncherUtilsTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/LauncherUtilsTest.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -40,6 +41,7 @@ import org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor;
import org.apache.heron.spi.utils.ReflectionUtils;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({ReflectionUtils.class, TopologyUtils.class, TopologyAPI.Topology.class})
public class LauncherUtilsTest {
@Test
diff --git a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/SchedulerUtilsTest.java b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/SchedulerUtilsTest.java
index 8173df4..a1b7c3b 100644
--- a/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/SchedulerUtilsTest.java
+++ b/heron/scheduler-core/tests/java/org/apache/heron/scheduler/utils/SchedulerUtilsTest.java
@@ -29,6 +29,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -44,6 +45,7 @@ import org.apache.heron.spi.utils.ShellUtils;
import static org.mockito.Mockito.eq;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({FileUtils.class, ShellUtils.class, SchedulerUtils.class})
public class SchedulerUtilsTest {
private static final String WORKING_DIR = "home";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraLauncherTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraLauncherTest.java
index 4e8bb2e..76e94a5 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraLauncherTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraLauncherTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -33,6 +34,7 @@ import org.apache.heron.spi.packing.PackingPlan;
import org.apache.heron.spi.scheduler.IScheduler;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(LauncherUtils.class)
public class AuroraLauncherTest {
@Test
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraSchedulerTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraSchedulerTest.java
index cd61916..b32bb22 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraSchedulerTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/aurora/AuroraSchedulerTest.java
@@ -37,6 +37,7 @@ import org.junit.runner.RunWith;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -66,6 +67,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({TokenSub.class, Config.class})
public class AuroraSchedulerTest {
private static final String AURORA_PATH = "path.aurora";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesLauncherTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesLauncherTest.java
index c211a77..1af0230 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesLauncherTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/kubernetes/KubernetesLauncherTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -33,6 +34,7 @@ import org.apache.heron.spi.packing.PackingPlan;
import org.apache.heron.spi.scheduler.IScheduler;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(LauncherUtils.class)
public class KubernetesLauncherTest {
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/local/LocalLauncherTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/local/LocalLauncherTest.java
index 3dd7347..ff6a890 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/local/LocalLauncherTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/local/LocalLauncherTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -34,6 +35,7 @@ import org.apache.heron.spi.common.Key;
import org.apache.heron.spi.packing.PackingPlan;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(LocalContext.class)
public class LocalLauncherTest {
private static final String TOPOLOGY_NAME = "testTopology";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonControllerTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonControllerTest.java
index 8bb038a..149f7e6 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonControllerTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonControllerTest.java
@@ -30,6 +30,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -37,6 +38,7 @@ import org.apache.heron.spi.utils.NetworkUtils;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(NetworkUtils.class)
public class MarathonControllerTest {
private static final String MARATHON_URI = "http://marathon.uri:8080";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonLauncherTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonLauncherTest.java
index 4c54902..394d9a0 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonLauncherTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/marathon/MarathonLauncherTest.java
@@ -24,6 +24,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -33,6 +34,7 @@ import org.apache.heron.spi.packing.PackingPlan;
import org.apache.heron.spi.scheduler.IScheduler;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(LauncherUtils.class)
public class MarathonLauncherTest {
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/nomad/NomadSchedulerTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/nomad/NomadSchedulerTest.java
index 276d6ff..1c0a43b 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/nomad/NomadSchedulerTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/nomad/NomadSchedulerTest.java
@@ -58,8 +58,8 @@ import org.apache.heron.spi.packing.Resource;
import static org.mockito.Matchers.anyVararg;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore({"javax.net.ssl.*", "jdk.internal.reflect.*"})
@PrepareForTest({NomadScheduler.class, Job.class, SchedulerUtils.class})
-@PowerMockIgnore("javax.net.ssl.*")
public class NomadSchedulerTest {
private static final Logger LOG = Logger.getLogger(NomadSchedulerTest.class.getName());
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmControllerTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmControllerTest.java
index 86adb18..a4dc0de 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmControllerTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmControllerTest.java
@@ -32,12 +32,14 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Matchers;
import org.mockito.Mockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.apache.heron.spi.utils.ShellUtils;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({SlurmContext.class, ShellUtils.class})
public class SlurmControllerTest {
private static final String WORKING_DIRECTORY = "workingDirectory";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmLauncherTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmLauncherTest.java
index 9cddb3e..ae86ee0 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmLauncherTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmLauncherTest.java
@@ -27,6 +27,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -37,6 +38,7 @@ import org.apache.heron.spi.packing.PackingPlan;
import org.apache.heron.spi.scheduler.IScheduler;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({SlurmContext.class, LauncherUtils.class})
public class SlurmLauncherTest {
private static final String TOPOLOGY_NAME = "testTopology";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmSchedulerTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmSchedulerTest.java
index be3ac54..533ce77 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmSchedulerTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/slurm/SlurmSchedulerTest.java
@@ -32,6 +32,7 @@ import org.junit.runner.RunWith;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -42,6 +43,7 @@ import org.apache.heron.spi.common.Key;
import org.apache.heron.spi.packing.PackingPlan;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({SlurmContext.class, SchedulerUtils.class})
public class SlurmSchedulerTest {
private static final String SLURM_PATH = "path.heron";
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronExecutorTaskTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronExecutorTaskTest.java
index 7d6e992..e4bd95c 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronExecutorTaskTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronExecutorTaskTest.java
@@ -48,7 +48,7 @@ import org.apache.heron.spi.utils.ShellUtils;
import org.apache.reef.runtime.common.files.REEFFileNames;
@RunWith(PowerMockRunner.class)
-@PowerMockIgnore("javax.swing.*")
+@PowerMockIgnore({"javax.swing.*", "jdk.internal.reflect.*"})
public class HeronExecutorTaskTest {
@Test
public void providesConfigsNeededForExecutorCmd() throws Exception {
diff --git a/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronMasterDriverTest.java b/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronMasterDriverTest.java
index f18fe3d..f191c30 100644
--- a/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronMasterDriverTest.java
+++ b/heron/schedulers/tests/java/org/apache/heron/scheduler/yarn/HeronMasterDriverTest.java
@@ -37,6 +37,7 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -74,6 +75,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
public class HeronMasterDriverTest {
private EvaluatorRequestor mockRequestor;
private HeronMasterDriver driver;
diff --git a/heron/simulator/src/java/BUILD b/heron/simulator/src/java/BUILD
index 84569b3..defae42 100644
--- a/heron/simulator/src/java/BUILD
+++ b/heron/simulator/src/java/BUILD
@@ -19,6 +19,7 @@ simulator_deps_files = \
"//heron/common/src/java:config-java",
"//heron/common/src/java:utils-java",
"//heron/instance/src/java:instance-java",
+ "@javax_xml_bind_jaxb//jar",
"@org_yaml_snakeyaml//jar",
]
diff --git a/heron/spi/src/java/org/apache/heron/spi/utils/ReflectionUtils.java b/heron/spi/src/java/org/apache/heron/spi/utils/ReflectionUtils.java
index 0ac9c0a..727a864 100644
--- a/heron/spi/src/java/org/apache/heron/spi/utils/ReflectionUtils.java
+++ b/heron/spi/src/java/org/apache/heron/spi/utils/ReflectionUtils.java
@@ -19,7 +19,9 @@
package org.apache.heron.spi.utils;
-public final class ReflectionUtils {
+public final class ReflectionUtils extends ClassLoader {
+ private static final ReflectionUtils REF = new ReflectionUtils();
+
private ReflectionUtils() {
}
@@ -36,6 +38,6 @@ public final class ReflectionUtils {
if (className == null) {
throw new ClassNotFoundException("Can not instantiate class. className must not be null");
}
- return (T) classLoader.loadClass(className).newInstance();
+ return (T) REF.loadClass(className, false).newInstance();
}
}
diff --git a/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java b/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java
index 5df902d..27ad696 100644
--- a/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java
+++ b/heron/spi/tests/java/org/apache/heron/spi/common/ConfigLoaderTest.java
@@ -28,6 +28,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -40,6 +41,7 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(ConfigLoader.class)
public class ConfigLoaderTest {
private static final String TEST_DATA_PATH =
diff --git a/heron/spi/tests/java/org/apache/heron/spi/common/ConfigTest.java b/heron/spi/tests/java/org/apache/heron/spi/common/ConfigTest.java
index 886e8b3..b7d5ed5 100644
--- a/heron/spi/tests/java/org/apache/heron/spi/common/ConfigTest.java
+++ b/heron/spi/tests/java/org/apache/heron/spi/common/ConfigTest.java
@@ -25,6 +25,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -32,8 +33,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(ConfigLoader.class)
-
public class ConfigTest {
private static final String TEST_DATA_PATH =
"/__main__/heron/spi/tests/java/org/apache/heron/spi/common/testdata";
diff --git a/heron/spi/tests/java/org/apache/heron/spi/utils/NetworkUtilsTest.java b/heron/spi/tests/java/org/apache/heron/spi/utils/NetworkUtilsTest.java
index e28e88d..a8410e8 100644
--- a/heron/spi/tests/java/org/apache/heron/spi/utils/NetworkUtilsTest.java
+++ b/heron/spi/tests/java/org/apache/heron/spi/utils/NetworkUtilsTest.java
@@ -42,6 +42,7 @@ import org.junit.runner.RunWith;
import org.mockito.Matchers;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -49,8 +50,8 @@ import org.apache.heron.common.basics.Pair;
import org.apache.heron.common.basics.SysUtils;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({
- SysUtils.class, NetworkUtils.class, ShellUtils.class})
+@PowerMockIgnore("jdk.internal.reflect.*")
+@PrepareForTest({SysUtils.class, NetworkUtils.class, ShellUtils.class})
public class NetworkUtilsTest {
private static final Logger LOG = Logger.getLogger(NetworkUtilsTest.class.getName());
diff --git a/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/dlog/DlogStorageTest.java b/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/dlog/DlogStorageTest.java
index 7780165..1451c6c 100644
--- a/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/dlog/DlogStorageTest.java
+++ b/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/dlog/DlogStorageTest.java
@@ -31,6 +31,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -58,6 +59,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({Namespace.class, CheckpointManager.InstanceStateCheckpoint.class})
public class DlogStorageTest {
diff --git a/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/hdfs/HDFSStorageTest.java b/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/hdfs/HDFSStorageTest.java
index ff65f6e..ee31dc4 100644
--- a/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/hdfs/HDFSStorageTest.java
+++ b/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/hdfs/HDFSStorageTest.java
@@ -29,6 +29,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -56,6 +57,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest({FileSystem.class, CheckpointManager.InstanceStateCheckpoint.class})
public class HDFSStorageTest {
diff --git a/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/localfs/LocalFileSystemStorageTest.java b/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/localfs/LocalFileSystemStorageTest.java
index e6d8481..4e60582 100644
--- a/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/localfs/LocalFileSystemStorageTest.java
+++ b/heron/statefulstorages/tests/java/org/apache/heron/statefulstorage/localfs/LocalFileSystemStorageTest.java
@@ -28,6 +28,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -49,6 +50,7 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(FileUtils.class)
public class LocalFileSystemStorageTest {
private PhysicalPlans.Instance instance;
diff --git a/heron/statemgrs/tests/java/org/apache/heron/statemgr/localfs/LocalFileSystemStateManagerTest.java b/heron/statemgrs/tests/java/org/apache/heron/statemgr/localfs/LocalFileSystemStateManagerTest.java
index 8419ac9..c3c5873 100644
--- a/heron/statemgrs/tests/java/org/apache/heron/statemgr/localfs/LocalFileSystemStateManagerTest.java
+++ b/heron/statemgrs/tests/java/org/apache/heron/statemgr/localfs/LocalFileSystemStateManagerTest.java
@@ -31,6 +31,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -62,6 +63,7 @@ import static org.mockito.Mockito.verify;
* LocalFileSystemStateManager Tester.
*/
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(FileUtils.class)
public class LocalFileSystemStateManagerTest {
diff --git a/heron/statemgrs/tests/java/org/apache/heron/statemgr/zookeeper/ZkUtilsTest.java b/heron/statemgrs/tests/java/org/apache/heron/statemgr/zookeeper/ZkUtilsTest.java
index 0ea1b4e..b017d73 100644
--- a/heron/statemgrs/tests/java/org/apache/heron/statemgr/zookeeper/ZkUtilsTest.java
+++ b/heron/statemgrs/tests/java/org/apache/heron/statemgr/zookeeper/ZkUtilsTest.java
@@ -26,6 +26,7 @@ import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -43,6 +44,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("jdk.internal.reflect.*")
@PrepareForTest(NetworkUtils.class)
public class ZkUtilsTest {
diff --git a/heron/tools/apiserver/src/java/BUILD b/heron/tools/apiserver/src/java/BUILD
index eda15bc..1bb91c3 100644
--- a/heron/tools/apiserver/src/java/BUILD
+++ b/heron/tools/apiserver/src/java/BUILD
@@ -44,11 +44,14 @@ apiserver_deps_files = \
packing_deps_files + \
uploader_deps_files + [
"//third_party/java:cli",
- "@org_yaml_snakeyaml//jar",
"//third_party/java:jetty-jersey-java",
"//third_party/java:commons-compress",
"//third_party/java:jackson",
"//third_party/java:logging",
+ "@javax_activation_activation//jar",
+ "@javax_annotation_javax_annotation_api//jar",
+ "@javax_xml_bind_jaxb//jar",
+ "@org_yaml_snakeyaml//jar",
]
java_binary(