You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2025/06/03 14:02:11 UTC
(hadoop-release-support) branch main updated: HADOOP-19565. Release Hadoop 3.4.2: improving release commands
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-release-support.git
The following commit(s) were added to refs/heads/main by this push:
new e818784 HADOOP-19565. Release Hadoop 3.4.2: improving release commands
e818784 is described below
commit e8187849d60c6cda8f30fc654d6fc34cca2c325e
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Jun 3 15:01:22 2025 +0100
HADOOP-19565. Release Hadoop 3.4.2: improving release commands
---
README.md | 29 ++++++++++++-----
build.xml | 37 +++++++---------------
pom.xml | 16 ++++++++--
src/releases/release-info-3.4.2.properties | 1 -
.../apache/hadoop/validator/TestRuntimeValid.java | 5 +--
5 files changed, 47 insertions(+), 41 deletions(-)
diff --git a/README.md b/README.md
index 895f6ca..8cfda36 100644
--- a/README.md
+++ b/README.md
@@ -469,10 +469,15 @@ locally.
# How to download and build a staged release candidate
-This relies on the release-info file pointing to the source directory
+This project can be used to download and validated a release created by other people,
+downloading the staged artifacts and validating their signatures before
+executing some (minimal) commands.
+
+This relies on the relevant `release-info-` file declaring the URL to download the artifacts from, and the maven staging repository.
+
```properties
-http.source=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.version}-RC${rc}/
+amd.src.dir=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.version}-RC${rc}/
```
### Targets of Relevance
@@ -490,10 +495,9 @@ http.source=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.versio
| `release.bin.commands` | execute a series of commands against the untarred binaries |
| `release.site.untar` | untar the downloaded site artifact |
| `release.site.validate` | perform minimal validation of the site. |
-| `release.arm.untar` | untar the ARM binary file |
-| `release.arm.commands` | execute commands against the arm binaries |
| `release.lean.tar` | create a release of the x86 binary tar without the AWS SDK |
+
set `check.native.binaries` to false to skip native binary checks on platforms without them
### Download the Staged RC files from the Apache http servers
@@ -539,11 +543,11 @@ in by declaring them in the property `source.compile.maven.args`
```properties
source.compile.maven.args=-Pnative
```
-These are added at the end of the hard-coded arguments (`clean install -DskipTests`)
+These are added at the end of the hard-coded arguments (`ant clean install -DskipTests`)
Testing is also possible through the target `release.src.test`
-```
+```bash
ant release.src.test
```
Again, the options set in `source.compile.maven.args` are passed down.
@@ -594,10 +598,19 @@ If `check.native.binaries` is false, the `bin/hadoop checknative`
is still executed, with the outcome printed (reporting a failure if
the binaries are not present).
-The and build itself is successful.
+The ant build itself will succeed, even if the `checknative` command reports a failure.
## Testing ARM binaries
+There are ARM variants of the commands to fetch and validate the ARM binaries.
+
+| target | action |
+|-------------------------|------------------------------------------------------------|
+| `release.fetch.arm` | fetch ARM artifacts |
+| `gpg.arm.verify` | verify ARM artifacts |
+| `release.arm.untar` | untar the ARM binary file |
+| `release.arm.commands` | execute commands against the arm binaries |
+
```bash
# untars the `-aarch64.tar.gz` binary
ant release.arm.untar
@@ -680,7 +693,6 @@ Review this to make sure there are no unexpected artifacts coming in,
ant cloudstore.build
```
-Note: this does not include the AWS V1 SDK `-Pextra` profile.
## Build and test Google GCS
@@ -747,6 +759,7 @@ ant parquet.test
```
+
# After the Vote Succeeds: publishing the release
## Update the announcement and create site/email notifications
diff --git a/build.xml b/build.xml
index 9c1c220..0441213 100644
--- a/build.xml
+++ b/build.xml
@@ -218,7 +218,6 @@
spark.version=${spark.version}
cloudstore.dir=${cloudstore.dir}
- cloudstore.profile=${cloudstore.profile}
bigdata-interop.dir=${bigdata-interop.dir}
hboss.dir=${hboss.dir}
cloud-examples.dir=${cloud-examples.dir}
@@ -248,7 +247,7 @@
<macrodef name="gpg-verify-file" >
<attribute name="dir" />
- <attribute name="file" />
+ <attribute name="name" />
<sequential>
<echo>Verifying GPG signature of @{dir}/@{name}</echo>
<gpg dir="@{dir}">
@@ -485,6 +484,13 @@
<gpgverify name="${release}-site.tar.gz"/>
<gpgverify name="${release}.tar.gz"/>
<gpgverify name="${release}-rat.txt"/>
+
+ </target>
+
+ <target name="gpg.arm.verify" depends="release.dir.check"
+ description="verify the downloaded arm artifacts">
+
+ <echo>Verifying GPG signatures of arm artifacts in ${release.dir}</echo>
<gpgverify name="${arm.binary.filename}"/>
</target>
@@ -940,11 +946,8 @@ Message is in file ${message.out}
if this is done with java11, it shouldn't be released.
</echo>
- <require p="cloudstore.profile"/>
-
<mvn dir="${cloudstore.dir}">
<arg value="-Psnapshots-and-staging"/>
- <arg value="-P${cloudstore.profile}"/>
<arg value="-Dhadoop.version=${hadoop.version}"/>
<arg value="clean"/>
<arg value="package"/>
@@ -952,26 +955,6 @@ Message is in file ${message.out}
</mvn>
</target>
- <!-- ========================================================= -->
- <!-- Hadoop FS API shim WiP -->
- <!-- ========================================================= -->
-
-
- <target name="fsapi.test" if="fs-api-shim.dir"
- depends="init"
- description="Build and test fs-api-shim">
- <echo>
- Build the fs api shim module.
- This MUST build against hadoop-3.2.0; the test version is
- what we want here.
- </echo>
- <mvn dir="${fs-api-shim.dir}">
- <arg value="-Psnapshots-and-staging"/>
- <arg value="-Dhadoop.test.version=${hadoop.version}"/>
- <arg value="clean"/>
- <arg value="test"/>
- </mvn>
- </target>
<!-- ========================================================= -->
<!-- Parquet -->
@@ -1120,7 +1103,6 @@ Message is in file ${message.out}
<download artifact="CHANGELOG.md"/>
<download artifact="RELEASENOTES.md"/>
<download artifact="${release}.tar.gz"/>
- <download artifact="${release}-aarch64.tar.gz"/>
<download artifact="${release}-site.tar.gz"/>
<download artifact="${release}-src.tar.gz"/>
<download artifact="${release}-rat.txt"/>
@@ -1224,6 +1206,9 @@ Message is in file ${message.out}
<echo>checknative</echo>
+ <echo>
+ Checking for native binaries; check.native.binaries=${check.native.binaries}
+ </echo>
<hadoopq failonerror="${check.native.binaries}">
<arg value="checknative" />
</hadoopq>
diff --git a/pom.xml b/pom.xml
index 9ee9681..3db252a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,8 +29,9 @@
<enforced.java.version>${javac.version}</enforced.java.version>
<maven-antrun-plugin.version>3.1.0</maven-antrun-plugin.version>
+ <aws-java-sdk-v2.version>2.29.52</aws-java-sdk-v2.version>
- <hadoop.version>3.4.0</hadoop.version>
+ <hadoop.version>3.4.2</hadoop.version>
</properties>
@@ -242,6 +243,17 @@
<version>${hadoop.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>bundle</artifactId>
+ <version>${aws-java-sdk-v2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -309,7 +321,7 @@
<profile>
<id>branch-3.4</id>
<properties>
- <hadoop.version>3.4.1</hadoop.version>
+ <hadoop.version>3.4.3-SNAPSHOT</hadoop.version>
</properties>
</profile>
diff --git a/src/releases/release-info-3.4.2.properties b/src/releases/release-info-3.4.2.properties
index cd1099d..a3656b2 100644
--- a/src/releases/release-info-3.4.2.properties
+++ b/src/releases/release-info-3.4.2.properties
@@ -28,5 +28,4 @@ amd.src.dir=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-3.4.2-RC1
arm.src.dir=${amd.src.dir}
http.source=${amd.src.dir}
asf.staging.url=https://repository.apache.org/content/repositories/orgapachehadoop-1437
-cloudstore.profile=sdk2
diff --git a/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java b/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java
index 87e56ad..ec3336a 100644
--- a/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java
+++ b/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java
@@ -22,10 +22,7 @@ import org.junit.Test;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.validator.CompileFS;
-
-/**
+import org.apache.hadoop.hdfs.DistributedFileSystem;/**
* Let's test that runtime.
*/
public class TestRuntimeValid {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org