You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/04/28 06:30:27 UTC
[ozone] branch master updated: HDDS-8269. [Snapshot] Enable CI native building and testing (#4596)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 94d8219eea HDDS-8269. [Snapshot] Enable CI native building and testing (#4596)
94d8219eea is described below
commit 94d8219eeacce5e69d6e9dd9e7302471638ea44b
Author: Swaminathan Balachandran <47...@users.noreply.github.com>
AuthorDate: Thu Apr 27 23:30:20 2023 -0700
HDDS-8269. [Snapshot] Enable CI native building and testing (#4596)
---
.github/workflows/ci.yml | 3 ++
dev-support/ci/selective_ci_checks.sh | 1 +
hadoop-hdds/pom.xml | 2 +-
hadoop-ozone/dev-support/checks/build.sh | 7 ++++
hadoop-ozone/dev-support/checks/native_check.sh | 50 +++++++++++++++++++++++++
hadoop-ozone/pom.xml | 2 +-
pom.xml | 2 +
7 files changed, 65 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 776142eba3..d63064e898 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -71,6 +71,8 @@ jobs:
steps:
- name: Checkout project
uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
- name: Cache for npm dependencies
uses: actions/cache@v3
with:
@@ -151,6 +153,7 @@ jobs:
run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx -Djavac.version=${{ matrix.java }}
env:
OZONE_WITH_COVERAGE: false
+ CANCEL_NATIVE_VERSION_CHECK: true
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
diff --git a/dev-support/ci/selective_ci_checks.sh b/dev-support/ci/selective_ci_checks.sh
index dd76189b1d..810685992a 100755
--- a/dev-support/ci/selective_ci_checks.sh
+++ b/dev-support/ci/selective_ci_checks.sh
@@ -307,6 +307,7 @@ function check_needs_build() {
start_end::group_start "Check if build is needed"
local pattern_array=(
"^hadoop-ozone/dev-support/checks/build.sh"
+ "^hadoop-ozone/dev-support/checks/native_check.sh"
"src/main/java"
"src/main/resources"
)
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 6735b2b978..211a60a632 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -252,7 +252,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-rocks-native</artifactId>
- <version>${hdds.version}</version>
+ <version>${hdds.rocks.native.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
diff --git a/hadoop-ozone/dev-support/checks/build.sh b/hadoop-ozone/dev-support/checks/build.sh
index 3189ada6fc..d55837b1d8 100755
--- a/hadoop-ozone/dev-support/checks/build.sh
+++ b/hadoop-ozone/dev-support/checks/build.sh
@@ -26,6 +26,13 @@ else
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
fi
+if [[ "${CANCEL_NATIVE_VERSION_CHECK}" != "true" ]]; then
+ NATIVE_MAVEN_OPTIONS="-Drocks_tools_native"
+ . "$DIR/native_check.sh"
+ init_native_maven_opts
+ MAVEN_OPTIONS="${MAVEN_OPTIONS} ${NATIVE_MAVEN_OPTIONS}"
+fi
export MAVEN_OPTS="-Xmx4096m $MAVEN_OPTS"
+echo "${MAVEN_OPTIONS}"
mvn ${MAVEN_OPTIONS} clean install "$@"
exit $?
diff --git a/hadoop-ozone/dev-support/checks/native_check.sh b/hadoop-ozone/dev-support/checks/native_check.sh
new file mode 100644
index 0000000000..44d6a9302a
--- /dev/null
+++ b/hadoop-ozone/dev-support/checks/native_check.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function get_rocks_native_git_sha() {
+ echo "Getting Rocks Native Git sha"
+ echo
+ # Getting the latest git hash for the ./hadoop-hdds/rocks-native directory
+ echo "git log -n 1 --format=\"%h\" ./hadoop-hdds/rocks-native"
+ ROCKS_NATIVE_GIT_SHA=$(git log -n 1 --format="%h" ./hadoop-hdds/rocks-native)
+ echo "ROCKS_NATIVE_GIT_SHA = ${ROCKS_NATIVE_GIT_SHA}"
+ readonly ROCKS_NATIVE_GIT_SHA
+}
+
+function init_native_maven_opts() {
+ get_rocks_native_git_sha
+ PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+ # Parsing out version number from project version by getting the first occurance of '-'.
+ # If project version is 1.4.0-SNAPSHOT, VERSION_NUMBER = 1.4.0
+ VERSION_NUMBER=$(echo "${PROJECT_VERSION}" | cut -f1 -d'-')
+ # Adding rocks native sha after the version number in the project version.
+ # EXPECTED_ROCK_NATIVE_VERSION = 1.4.0-<rocks native git sha>-SNAPSHOT
+ EXPECTED_ROCKS_NATIVE_VERSION=${VERSION_NUMBER}"-${ROCKS_NATIVE_GIT_SHA}"${PROJECT_VERSION:${#VERSION_NUMBER}}
+ echo "Checking Maven repo contains hdds-rocks-native of version ${EXPECTED_ROCKS_NATIVE_VERSION}"
+ mvn --non-recursive dependency:get -Dartifact=org.apache.ozone:hdds-rocks-native:${EXPECTED_ROCKS_NATIVE_VERSION} -q
+
+ MVN_GET_ROCKS_NATIVE_EXIT_CODE=$?
+ if [[ "${MVN_GET_ROCKS_NATIVE_EXIT_CODE}" == "0" ]]; then
+ echo "Using existing hdds-rocks-native artifact version: ${EXPECTED_ROCKS_NATIVE_VERSION}"
+ NATIVE_MAVEN_OPTIONS="-Dhdds.rocks.native.version=${EXPECTED_ROCKS_NATIVE_VERSION}"
+ else
+ echo "Building hdds-rocks-native from scratch as version ${EXPECTED_ROCKS_NATIVE_VERSION} was not found in the given Maven repos"
+ NATIVE_MAVEN_OPTIONS="-Drocks_tools_native"
+ fi
+ readonly NATIVE_MAVEN_OPTIONS
+ echo "Native Maven options : ${NATIVE_MAVEN_OPTIONS}"
+}
+
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 1984196d45..7d38e980ed 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -271,7 +271,7 @@
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-rocks-native</artifactId>
- <version>${hdds.version}</version>
+ <version>${hdds.rocks.native.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
diff --git a/pom.xml b/pom.xml
index 1533463d82..1617edbb5b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<declared.hdds.version>${hdds.version}</declared.hdds.version>
<declared.ozone.version>${ozone.version}</declared.ozone.version>
+ <!-- HDDS Rocks Native dependency version-->
+ <hdds.rocks.native.version>${hdds.version}</hdds.rocks.native.version>
<!-- Apache Ratis version -->
<ratis.version>2.4.2-8b8bdda-SNAPSHOT</ratis.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org