You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/07/02 04:35:13 UTC
[arrow] branch master updated: ARROW-5466: [Java][CI] Dockerize
Java CI, run all JDK builds in single Travis entry
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new e02ec4e ARROW-5466: [Java][CI] Dockerize Java CI, run all JDK builds in single Travis entry
e02ec4e is described below
commit e02ec4e80aec3c092a72890117574f34b931de30
Author: Wes McKinney <we...@apache.org>
AuthorDate: Mon Jul 1 23:35:03 2019 -0500
ARROW-5466: [Java][CI] Dockerize Java CI, run all JDK builds in single Travis entry
Since OpenJDK9 has been superseded by OpenJDK11, it is not available in package repositories, so I'm not sure it's worth maintaining a build for this.
I have pushed a pre-built Docker image for this to
https://cloud.docker.com/u/ursalab/repository/docker/ursalab/arrow-ci-java-all-jdks
Author: Wes McKinney <we...@apache.org>
Closes #4761 from wesm/java-dockerify and squashes the following commits:
8b8103729 <Wes McKinney> Actually run Java unit tests
cf568a28a <Wes McKinney> Code review feedback
6cb5e3f70 <Wes McKinney> Build Javadoc in docker-compose job
d9c390092 <Wes McKinney> Run all Java builds in a single Dockerized Travis CI entry
---
.travis.yml | 26 +++---------------
ci/docker_build_java.sh | 17 ++++++++++--
...s_script_javadoc.sh => docker_java_test_all.sh} | 16 ++++++-----
docker-compose.yml | 12 +++++++++
.../Dockerfile.all-jdks | 31 +++++++++++-----------
5 files changed, 56 insertions(+), 46 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 5dc9015..117f5dc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -199,32 +199,14 @@ matrix:
- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker-compose pull python-manylinux1; fi
script:
- if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh; fi
- - name: "Java w/ OpenJDK 8"
- language: java
- os: linux
- jdk: openjdk8
- before_script:
- - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
- - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_javadoc.sh
- - name: "Java w/ OpenJDK 9"
- language: java
- os: linux
- jdk: openjdk9
- before_script:
- - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
- script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
- - name: "Java w/ OpenJDK 11"
- language: java
+ - name: "Java OpenJDK8 and OpenJDK11"
+ language: cpp
os: linux
- jdk: openjdk11
before_script:
- if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+ - docker-compose pull java-all-jdks
script:
- - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
+ - docker-compose run java-all-jdks
- name: "Integration w/ OpenJDK 8, conda-forge toolchain"
language: java
os: linux
diff --git a/ci/docker_build_java.sh b/ci/docker_build_java.sh
index f3dd3f1..e6516b7 100755
--- a/ci/docker_build_java.sh
+++ b/ci/docker_build_java.sh
@@ -25,10 +25,23 @@ mkdir -p /build/java
arrow_src=/build/java/arrow
+# Remove any pre-existing artifacts
+rm -rf $arrow_src
+
pushd /arrow
- rsync -a header java format integration $arrow_src
+rsync -a header java format integration $arrow_src
popd
+JAVA_ARGS=
+if [ "$ARROW_JAVA_RUN_TESTS" != "1" ]; then
+ JAVA_ARGS=-DskipTests
+fi
+
pushd $arrow_src/java
- mvn -B -DskipTests -Drat.skip=true install
+mvn -B $JAVA_ARGS -Drat.skip=true install
+
+if [ "$ARROW_JAVADOC" == "1" ]; then
+ export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
+ mvn -B site
+fi
popd
diff --git a/ci/travis_script_javadoc.sh b/ci/docker_java_test_all.sh
similarity index 69%
rename from ci/travis_script_javadoc.sh
rename to ci/docker_java_test_all.sh
index 755d462..1466907 100755
--- a/ci/travis_script_javadoc.sh
+++ b/ci/docker_java_test_all.sh
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -19,13 +18,16 @@
set -e
-source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-JAVA_DIR=${TRAVIS_BUILD_DIR}/java
+export ARROW_TEST_DATA=/arrow/testing/data
-pushd $JAVA_DIR
+export ARROW_JAVA_RUN_TESTS=1
-export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=warn"
-$TRAVIS_MVN -B site
+export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+export ARROW_JAVADOC=1
+bash $SOURCE_DIR/docker_build_java.sh
-popd
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+export ARROW_JAVADOC=0
+bash $SOURCE_DIR/docker_build_java.sh
diff --git a/docker-compose.yml b/docker-compose.yml
index 2fa5ab4..9417148 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -292,6 +292,18 @@ services:
- .:/arrow:ro # ensures that docker won't contaminate the host directory
- maven-cache:/root/.m2:delegated
+ java-all-jdks:
+ # Usage:
+ # docker-compose build java-all-jdks
+ # docker-compose run java-all-jdks
+ image: ursalab/arrow-ci-java-all-jdks:latest
+ build:
+ context: .
+ dockerfile: java/Dockerfile.all-jdks
+ volumes:
+ - .:/arrow:ro # ensures that docker won't contaminate the host directory
+ - maven-cache:/root/.m2:delegated
+
js:
image: arrow:js
build:
diff --git a/ci/docker_build_java.sh b/java/Dockerfile.all-jdks
old mode 100755
new mode 100644
similarity index 62%
copy from ci/docker_build_java.sh
copy to java/Dockerfile.all-jdks
index f3dd3f1..bf4e2af
--- a/ci/docker_build_java.sh
+++ b/java/Dockerfile.all-jdks
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,19 +15,21 @@
# specific language governing permissions and limitations
# under the License.
-set -e
+FROM ubuntu:18.04
-export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+# install build essentials
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ wget \
+ software-properties-common \
+ ca-certificates \
+ maven \
+ rsync \
+ tzdata \
+ openjdk-8-jdk \
+ openjdk-11-jdk && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
-# /arrow/java is read-only
-mkdir -p /build/java
-
-arrow_src=/build/java/arrow
-
-pushd /arrow
- rsync -a header java format integration $arrow_src
-popd
-
-pushd $arrow_src/java
- mvn -B -DskipTests -Drat.skip=true install
-popd
+# Test all supported JDKs
+CMD ["arrow/ci/docker_java_test_all.sh"]