You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by li...@apache.org on 2020/03/08 03:22:55 UTC
[submarine] branch master updated: SUBMARINE-406. optimize travis
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 33a1659 SUBMARINE-406. optimize travis
33a1659 is described below
commit 33a165950b7cff23fb1be69674011136a99fc98e
Author: Xun Liu <li...@apache.org>
AuthorDate: Fri Mar 6 16:35:54 2020 +0800
SUBMARINE-406. optimize travis
### What is this PR for?
1. Test cases are not fully covered in travis, and there is a problem of repeated tests. So we need to optimize and refactor travis test cases.
2. Remove duplicate testing tasks in travis.
3. Reduced test task time in travis.
### What type of PR is it?
[Improvement | Refactoring]
### What is the Jira issue?
* https://issues.apache.org/jira/browse/SUBMARINE-406
### How should this be tested?
* https://travis-ci.org/liuxunorg/submarine/builds/659562164
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Xun Liu <li...@apache.org>
Closes #207 from liuxunorg/SUBMARINE-406 and squashes the following commits:
fba7d8c [Xun Liu] SUBMARINE-406. optimize travis
---
.travis.yml | 150 ++++++++++++---------
submarine-server/server-core/pom.xml | 2 +-
.../server-submitter/submitter-k8s/pom.xml | 2 +-
.../server-submitter/submitter-yarn/pom.xml | 4 +-
4 files changed, 92 insertions(+), 66 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 6e11546..e6266b9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -43,13 +43,20 @@ services:
env:
global:
# If you need to compile Phadoop-3.1 or Phadoop-3.2, you need to add `!submarine-server/server-submitter/submitter-yarnservice` in EXCLUDE_SUBMARINE
- - EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-core,!submarine-server/server-rpc"
- - EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-web-ng"
+ - SUBMARINE="org.apache.submarine"
+ - EXCLUDE_SUBMARINE="!${SUBMARINE}:submarine-all,!${SUBMARINE}:submarine-client,!${SUBMARINE}:submarine-dist"
+ - EXCLUDE_SERVER="!${SUBMARINE}:server-api,!${SUBMARINE}:server-core,!${SUBMARINE}:server-rpc"
+ - EXCLUDE_CLIENT="!${SUBMARINE}:submarine-client"
+ - EXCLUDE_ALL="!${SUBMARINE}:submarine-all"
+ - EXCLUDE_WORKBENCH="!${SUBMARINE}:submarine-workbench,!${SUBMARINE}:workbench-web,!${SUBMARINE}:workbench-web-ng"
- EXCLUDE_INTERPRETER="" # Template disable by SUBMARINE-381 "!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter
- - EXCLUDE_K8S="!submarine-server/server-submitter/submitter-k8s"
- - EXCLUDE_COMMON_RPC="!submarine-commons/commons-rpc"
- - EXCLUDE_COMMONS="${EXCLUDE_COMMON_RPC},!submarine-commons/commons-utils,!submarine-commons/commons-cluster,!submarine-commons/commons-runtime,!submarine-commons/commons-metastore"
- - EXCLUDE_CLOUD="!org.apache.submarine:submarine-cloud"
+ - EXCLUDE_SUBMITTER_K8S="!${SUBMARINE}:submitter-k8s"
+ - EXCLUDE_SUBMITTER_YARN="!${SUBMARINE}:submitter-yarn"
+ - EXCLUDE_SUBMITTER="!${SUBMARINE}:server-submitter,${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMITTER_YARN}"
+ - EXCLUDE_COMMONS="!${SUBMARINE}:commons-cluster,!${SUBMARINE}:commons-metastore,!${SUBMARINE}:commons-rpc,!${SUBMARINE}:commons-runtime,!${SUBMARINE}:commons-utils"
+ - EXCLUDE_CLOUD="!${SUBMARINE}:submarine-cloud"
+ - EXCLUDE_DIST="!${SUBMARINE}:submarine-dist"
+ - EXCLUDE_TEST="!${SUBMARINE}:submarine-test,!${SUBMARINE}:test-e2e,!${SUBMARINE}:test-k8s"
- MOZ_HEADLESS=1
before_install:
@@ -96,31 +103,81 @@ matrix:
dist: xenial
env: PROFILE="" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
- - name: Build submarine submitter on hadoop-2.9 (default)
+ - name: Test submarine test-k8s
+ dist: xenial
+ services: docker
+ language: java
+ jdk: openjdk8
+ addons:
+ chrome: stable
+ firefox: latest
+ # Because submarine on k8s uses port 80, it needs to be set `-Durl=http://127.0.0.1`
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1" MODULES="" TEST_MODULES="-pl ${EXCLUDE_SERVER},${EXCLUDE_COMMONS},${EXCLUDE_CLIENT},org.apache.submarine:test-k8s" TEST_PROJECTS=""
+
+ - name: Test submarine test-e2e
+ language: java
+ jdk: openjdk8
+ dist: xenial
+ addons:
+ chrome: stable
+ firefox: latest
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am" TEST_MODULES="-pl org.apache.submarine:test-e2e" TEST_PROJECTS=""
+
+ - name: Test submarine commons-cluster
+ language: java
+ jdk: openjdk8
+ dist: xenial
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" TEST_MODULES="-pl org.apache.submarine:commons-cluster" TEST_PROJECTS=""
+
+ - name: Test submarine commons-metastore
language: java
jdk: openjdk8
dist: xenial
- env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},!submarine-dist" TEST_MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMON_RPC},!submarine-dist" TEST_PROJECTS=""
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" TEST_MODULES="-pl org.apache.submarine:commons-metastore" TEST_PROJECTS=""
- - name: Build submarine submitter on hadoop-2.10
+ - name: Test submarine commons-rpc
language: java
jdk: openjdk8
dist: xenial
- env: PROFILE="-Phadoop-2.10" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},!submarine-dist" TEST_MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMON_RPC},!submarine-dist" TEST_PROJECTS=""
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" TEST_MODULES="-pl org.apache.submarine:commons-rpc" TEST_PROJECTS=""
- - name: Build submarine submitter on hadoop-3.1
+ - name: Test submarine commons-runtime
language: java
jdk: openjdk8
dist: xenial
- env: PROFILE="-Phadoop-3.1" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},!submarine-dist" TEST_MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMON_RPC},!submarine-dist" TEST_PROJECTS=""
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_COMMONS},${EXCLUDE_SUBMITTER},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLIENT},${EXCLUDE_CLOUD},${EXCLUDE_SERVER},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" TEST_MODULES="-pl org.apache.submarine:commons-runtime" TEST_PROJECTS=""
- - name: Build submarine submitter on hadoop-3.2
+ - name: Test submarine server
language: java
jdk: openjdk8
dist: xenial
- env: PROFILE="-Phadoop-3.2" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},!submarine-dist" TEST_MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMON_RPC},!submarine-dist" TEST_PROJECTS=""
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_ALL},${EXCLUDE_DIST},${EXCLUDE_TEST}" TEST_MODULES="-pl ${EXCLUDE_COMMONS},org.apache.submarine:server-core" TEST_PROJECTS=""
- - name: Build submarine submitter on Kubernetes
+ - name: Test submarine submitter on hadoop-2.9 (default)
+ language: java
+ jdk: openjdk8
+ dist: xenial
+ env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER}" TEST_PROJECTS=""
+
+ - name: Test submarine submitter on hadoop-2.10
+ language: java
+ jdk: openjdk8
+ dist: xenial
+ env: PROFILE="-Phadoop-2.10" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER}" TEST_PROJECTS=""
+
+ - name: Test submarine submitter on hadoop-3.1
+ language: java
+ jdk: openjdk8
+ dist: xenial
+ env: PROFILE="-Phadoop-3.1" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER}" TEST_PROJECTS=""
+
+ - name: Test submarine submitter on hadoop-3.2
+ language: java
+ jdk: openjdk8
+ dist: xenial
+ env: PROFILE="-Phadoop-3.2" BUILD_FLAG="clean package install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_DIST}" TEST_MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER},${EXCLUDE_CLOUD},${EXCLUDE_COMMONS},${EXCLUDE_DIST},${EXCLUDE_TEST},${EXCLUDE_ALL},${EXCLUDE_SERVER}" TEST_PROJECTS=""
+
+ - name: Test submarine submitter on Kubernetes
dist: xenial
services: docker
language: java
@@ -146,46 +203,14 @@ matrix:
- kubectl apply -f ./dev-support/travis/tf-operator/crd_v1.yaml
- kubectl apply -f ./dev-support/travis/tf-operator/tfevent-volume/.
- - name: Build submarine workbench-web VUE
- language: node_js
- node_js:
- - 10.15.0
- before_install:
- - cd submarine-workbench/workbench-web
- install:
- - yarn install
- script:
- - yarn
- - yarn run lint --no-fix && yarn run build
- env: NAME="Build workbench-web"
-
- - name: Test submarine test-e2e
- language: java
- jdk: openjdk8
- dist: xenial
- addons:
- chrome: stable
- firefox: latest
- env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am" TEST_MODULES="-pl submarine-test/test-e2e" TEST_PROJECTS=""
-
- - name: Test submarine-test-k8s
- dist: xenial
- services: docker
- language: java
- jdk: openjdk8
- addons:
- chrome: stable
- # Because submarine on k8s uses port 80, it needs to be set `-Durl=http://127.0.0.1`
- env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="verify -DskipRat -am -Durl=http://127.0.0.1" MODULES="" TEST_MODULES="-pl ${EXCLUDE_COMMONS},!org.apache.submarine:submarine-client,!org.apache.submarine:submitter-yarn,!org.apache.submarine:server-api,!org.apache.submarine:server-rpc,!org.apache.submarine:server-core,submarine-test/test-k8s" TEST_PROJECTS=""
-
# Template disable by SUBMARINE-381
#- name: Test submarine interpreter
# language: java
# jdk: "openjdk8"
# dist: xenial
- # env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH}" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
+ # env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl ${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH}" TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
- - name: Test submarine-sdk
+ - name: Test submarine sdk
language: python
python: 3.6
dist: xenial
@@ -197,11 +222,18 @@ matrix:
- ./submarine-sdk/pysubmarine/travis/lint.sh
- pytest --cov=submarine -vs
- - name: Test submarine distribution
- language: java
- jdk: openjdk8
- dist: xenial
- env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat" MODULES="-pl ${EXCLUDE_K8S}" TEST_MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_COMMON_RPC}" TEST_PROJECTS=""
+ - name: Test submarine workbench-web VUE
+ language: node_js
+ node_js:
+ - 10.15.0
+ before_install:
+ - cd submarine-workbench/workbench-web
+ install:
+ - yarn install
+ script:
+ - yarn
+ - yarn run lint --no-fix && yarn run build
+ env: NAME="Build workbench-web"
- name: Test submarine workbench-web Angular
language: node_js
@@ -221,15 +253,9 @@ matrix:
- npm run e2e -- --protractor-config=e2e/protractor-ci.conf.js
env: BUILD_FLAG="clean package -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl org.apache.submarine:workbench-web-ng" TEST_MODULES="-pl org.apache.submarine:workbench-web-ng" TEST_PROJECTS=""
- - name: Test submarine-server
- language: java
- jdk: openjdk8
- dist: xenial
- env: PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install -DskipTests" TEST_FLAG="test -DskipRat" MODULES="-pl ${EXCLUDE_K8S},${EXCLUDE_WORKBENCH},${EXCLUDE_INTERPRETER}" TEST_MODULES="-pl org.apache.submarine:server-core" TEST_PROJECTS=""
-
install:
- mvn --version
- - echo "install-> mvn $BUILD_FLAG $MODULES $PROFILE -B"
+ - echo ">>> mvn $BUILD_FLAG $MODULES $PROFILE -B"
- mvn $BUILD_FLAG $MODULES $PROFILE -B
before_script:
@@ -238,7 +264,7 @@ before_script:
script:
- if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
- - echo "script-> mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
+ - echo ">>> mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS"
- mvn $TEST_FLAG $MODULES $PROFILE -B $TEST_PROJECTS
after_success:
diff --git a/submarine-server/server-core/pom.xml b/submarine-server/server-core/pom.xml
index 4cd27b3..a9dc861 100644
--- a/submarine-server/server-core/pom.xml
+++ b/submarine-server/server-core/pom.xml
@@ -405,7 +405,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/submarine-server/server-submitter/submitter-k8s/pom.xml b/submarine-server/server-submitter/submitter-k8s/pom.xml
index eabf49c..1181eee 100644
--- a/submarine-server/server-submitter/submitter-k8s/pom.xml
+++ b/submarine-server/server-submitter/submitter-k8s/pom.xml
@@ -65,7 +65,7 @@
</dependencies>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git a/submarine-server/server-submitter/submitter-yarn/pom.xml b/submarine-server/server-submitter/submitter-yarn/pom.xml
index d1b90e8..49faede 100644
--- a/submarine-server/server-submitter/submitter-yarn/pom.xml
+++ b/submarine-server/server-submitter/submitter-yarn/pom.xml
@@ -329,7 +329,7 @@
</profiles>
<build>
- <finalName>submarine-${artifactId}-${project.version}</finalName>
+ <finalName>submarine-${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -342,7 +342,7 @@
<goal>shade</goal>
</goals>
<configuration>
- <outputFile>target/submarine-${artifactId}-${project.version}-shade.jar</outputFile>
+ <outputFile>target/submarine-${project.artifactId}-${project.version}-shade.jar</outputFile>
<filters>
<filter>
<artifact>*:*</artifact>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org