You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2020/06/04 12:01:19 UTC
[hadoop-ozone] 01/18: collect coverage data from basic acceptance
tests
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch acceptance-coverage
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 1d744f2bec1cef359f3408e8c4def124bb8bea03
Author: Elek Márton <el...@apache.org>
AuthorDate: Thu May 28 16:00:15 2020 +0200
collect coverage data from basic acceptance tests
---
.github/workflows/post-commit.yml | 180 ---------------------
hadoop-ozone/dev-support/checks/acceptance.sh | 4 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dist/pom.xml | 42 +++++
.../src/main/compose/ozone/docker-compose.yaml | 5 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 0
hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +
7 files changed, 53 insertions(+), 182 deletions(-)
diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 5a920e5..6823032 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -24,78 +24,6 @@ jobs:
- uses: ./.github/buildenv
with:
args: ./hadoop-ozone/dev-support/checks/build.sh
- rat:
- name: rat
- runs-on: ubuntu-18.04
- steps:
- - uses: actions/checkout@master
- - uses: ./.github/buildenv
- with:
- args: ./hadoop-ozone/dev-support/checks/rat.sh
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: rat
- path: target/rat
- author:
- name: author
- runs-on: ubuntu-18.04
- steps:
- - uses: actions/checkout@master
- - uses: ./.github/buildenv
- with:
- args: ./hadoop-ozone/dev-support/checks/author.sh
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: author
- path: target/author
- unit:
- name: unit
- runs-on: ubuntu-18.04
- steps:
- - uses: actions/checkout@master
- - uses: ./.github/buildenv
- with:
- args: ./hadoop-ozone/dev-support/checks/unit.sh
- - uses: ./.github/buildenv
- if: github.repository == 'apache/hadoop-ozone'
- with:
- args: ./hadoop-ozone/dev-support/checks/sonar.sh
- env:
- SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: unit
- path: target/unit
- checkstyle:
- name: checkstyle
- runs-on: ubuntu-18.04
- steps:
- - uses: actions/checkout@master
- - uses: ./.github/buildenv
- with:
- args: ./hadoop-ozone/dev-support/checks/checkstyle.sh
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: checkstyle
- path: target/checkstyle
- findbugs:
- name: findbugs
- runs-on: ubuntu-18.04
- steps:
- - uses: actions/checkout@master
- - uses: ./.github/buildenv
- with:
- args: ./hadoop-ozone/dev-support/checks/findbugs.sh
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: findbugs
- path: target/findbugs
acceptance:
name: acceptance
runs-on: ubuntu-18.04
@@ -119,111 +47,3 @@ jobs:
with:
name: acceptance
path: /mnt/ozone/compose/result
- it-freon:
- name: it-freon
- runs-on: ubuntu-18.04
- needs:
- - build
- steps:
- - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- - uses: actions/checkout@master
- with:
- path: mnt/ozone
- - uses: ./mnt/ozone/.github/buildenv
- with:
- args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfreon
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: it-freon
- path: mnt/ozone/target/integration
- it-filesystem:
- name: it-filesystem
- runs-on: ubuntu-18.04
- needs:
- - build
- steps:
- - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- - uses: actions/checkout@master
- with:
- path: mnt/ozone
- - uses: ./mnt/ozone/.github/buildenv
- with:
- args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: it-filesystem
- path: mnt/ozone/target/integration
- it-filesystem-contract:
- name: it-filesystem-contract
- runs-on: ubuntu-18.04
- needs:
- - build
- steps:
- - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- - uses: actions/checkout@master
- with:
- path: mnt/ozone
- - uses: ./mnt/ozone/.github/buildenv
- with:
- args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pfilesystem-contract
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: it-filesystem-contract
- path: mnt/ozone/target/integration
- it-client:
- name: it-client
- runs-on: ubuntu-18.04
- needs:
- - build
- steps:
- - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- - uses: actions/checkout@master
- with:
- path: mnt/ozone
- - uses: ./mnt/ozone/.github/buildenv
- with:
- args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pclient
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: it-client
- path: mnt/ozone/target/integration
- it-hdds-om:
- name: it-hdds-om
- runs-on: ubuntu-18.04
- needs:
- - build
- steps:
- - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- - uses: actions/checkout@master
- with:
- path: mnt/ozone
- - uses: ./mnt/ozone/.github/buildenv
- with:
- args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Phdds-om
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: it-hdds-om
- path: mnt/ozone/target/integration
- it-ozone:
- name: it-ozone
- runs-on: ubuntu-18.04
- needs:
- - build
- steps:
- - run: sudo mkdir mnt && sudo mount --bind /mnt `pwd`/mnt && sudo chmod 777 mnt
- - uses: actions/checkout@master
- with:
- path: mnt/ozone
- - uses: ./mnt/ozone/.github/buildenv
- with:
- args: ./mnt/ozone/hadoop-ozone/dev-support/checks/integration.sh -Pozone
- - uses: actions/upload-artifact@master
- if: always()
- with:
- name: it-ozone
- path: mnt/ozone/target/integration
\ No newline at end of file
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
index c3c1aa0..30259c7 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -24,9 +24,11 @@ DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
if [ ! -d "$DIST_DIR" ]; then
echo "Distribution dir is missing. Doing a full build"
- "$DIR/build.sh"
+ "$DIR/build.sh" -Pjacoco
fi
+export HADOOP_OPTS='-javaagent:/opt/hadoop/share/jacoco/jacoco-agent.jar=destfile=/tmp/jacoco.exec,includes=org.apache.hadoop.ozone.*:org.apache.hadoop.hdds'
+
cd "$DIST_DIR/compose" || exit 1
./test-all.sh
RES=$?
diff --git a/hadoop-ozone/dev-support/checks/build.sh b/hadoop-ozone/dev-support/checks/build.sh
index f461da1..2cdc4fe 100755
--- a/hadoop-ozone/dev-support/checks/build.sh
+++ b/hadoop-ozone/dev-support/checks/build.sh
@@ -17,5 +17,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
export MAVEN_OPTS="-Xmx4096m"
-mvn -B -Dmaven.javadoc.skip=true -DskipTests clean install
+mvn -B -Dmaven.javadoc.skip=true -DskipTests clean install "$@"
exit $?
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index 896831b..752a8c5 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -379,7 +379,49 @@
<artifactId>hadoop-ozone-recon</artifactId>
</dependency>
</dependencies>
+ </profile>
+ <profile>
+ <id>jacoco</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <id>copy-jacoco-files</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>
+ target/ozone-${ozone.version}/share/jacoco
+ </outputDirectory>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.jacoco</groupId>
+ <artifactId>org.jacoco.agent</artifactId>
+ <version>0.8.5</version>
+ <classifier>runtime</classifier>
+ <destFileName>jacoco-agent.jar</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.jacoco</groupId>
+ <artifactId>org.jacoco.cli</artifactId>
+ <version>0.8.5</version>
+ <classifier>nodeps</classifier>
+ <destFileName>jacoco-cli.jar</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</profile>
<profile>
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
index 869aa05..2597612 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
@@ -37,11 +37,13 @@ services:
- 9882
environment:
<<: *replication
+ HADOOP_OPTS: ${HADOOP_OPTS}
command: ["ozone","datanode"]
om:
<<: *common-config
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ HADOOP_OPTS: ${HADOOP_OPTS}
<<: *replication
ports:
- 9874:9874
@@ -54,11 +56,13 @@ services:
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+ HADOOP_OPTS: ${HADOOP_OPTS}
<<: *replication
command: ["ozone","scm"]
s3g:
<<: *common-config
environment:
+ HADOOP_OPTS: ${HADOOP_OPTS}
<<: *replication
ports:
- 9878:9878
@@ -68,5 +72,6 @@ services:
ports:
- 9888:9888
environment:
+ HADOOP_OPTS: ${HADOOP_OPTS}
<<: *replication
command: ["ozone","recon"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
old mode 100755
new mode 100644
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
old mode 100755
new mode 100644
index 064b19d..5948ac9
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -108,7 +108,9 @@ execute_robot_test(){
local -i rc=$?
FULL_CONTAINER_NAME=$(docker-compose -f "$COMPOSE_FILE" ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
+
docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
+ docker cp "$FULL_CONTAINER_NAME:/tmp/jacoco.exec" "$RESULT_DIR/$OUTPUT_NAME.jacoco.exec" || true
copy_daemon_logs
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org