(hadoop-release-support) branch main updated: HADOOP-19087 build a lean tar

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stevel pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-release-support.git


The following commit(s) were added to refs/heads/main by this push:
     new 57dffde  HADOOP-19087 build a lean tar
57dffde is described below

commit 57dffded1c08556dfa28947d8eca76d6bd8f4cc8
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Tue Oct 8 16:39:34 2024 +0100

    HADOOP-19087 build a lean tar
    
    New target release.lean.tar expands the normal
    x86 binary tarball; deletes bundle.jar and
    archives/compresses it to a -lean.tar.gz
    with new signature and sha512 to match
---
 README.md | 36 +++++++++++++++++++++++-
 build.xml | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 125 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 889700d..0dd59ac 100644
--- a/README.md
+++ b/README.md
@@ -306,6 +306,40 @@ ant copy-scp-artifacts release.dir.check
 
 The `release.dir.check` target just lists the directory.
 
+### Build a lean binary tar
+
+The normal binary tar.gz files huge because they install a version of the AWS v2 SDK "bundle.jar"
+file which has been validated with the hadoop-aws module and the S3A connector which was built against it.
+
+This is a really big file because it includes all the "shaded" dependencies as well as client libraries
+to talk with many unused AWS services up to and including scheduling satellite downlink time.
+
+We ship the full bundle jar as it allows Hadoop and its downstream applications to be isolated from
+the choice of JAR dependencies in the AWS SDK. That is: it ensures a classpath that works out the box
+and stop having to upgrade on a schedule determined by maintains the AWS SDK pom files.
+
+It does make for big images and that has some negative consequences.
+* More data to download when installing Hadoop.
+* More space is required in the fileystem of any host into which it is installed.
+* Slower times to launch docker containers if installing the binary tar as part of the container launch
+  process.
+* Larger container images if preinstalled.
+
+The "lean" x86 binary tar.gz file aims to reduce eliminate these negative issues by being
+a variant of the normal x86 binary distribution with the relevant AWS SDK jar removed.
+
+The build target `release.lean.tar` can do this once the normal x86 binaries have been downloaded.
+
+```bash
+ant release.lean.tar
+```
+
+It performs the following actions:
+1. expands the binary .tar.gz under the path `target/bin-lean`
+2. deletes all files `bundle-*` from this expanded SDK
+3. builds a new binary release with the suffix `-lean.tar.gz`
+4. Generates new checksum and signatures.
+
 ### Building Arm64 binaries
 
 If arm64 binaries are being created then they must be
@@ -456,7 +490,7 @@ http.source=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.versio
 | `release.site.validate` | perform minimal validation of the site.                    |
 | `release.arm.untar`     | untar the ARM binary file                                  |
 | `release.arm.commands`  | execute commands against the arm binaries                  |
-|                         |                                                            |
+| `release.lean.tar`      | create a release of the x86 binary tar without the AWS SDK |
 
 set `check.native.binaries` to false to skip native binary checks on platforms without them
 
diff --git a/build.xml b/build.xml
index 6921428..e5edf86 100644
--- a/build.xml
+++ b/build.xml
@@ -84,6 +84,19 @@
     </sequential>
   </macrodef>
 
+  <macrodef name="sha512">
+    <attribute name="file" />
+    <sequential>
+      <checksum
+        algorithm="SHA-512"
+        fileext=".sha512"
+        file="@{file}"
+        pattern="SHA512 ({1}) = {0}"
+        forceoverwrite="true"
+        />
+    </sequential>
+  </macrodef>
+
 
   <!-- set immutable properties for this build -->
   <setpath name="downloads.dir" location="downloads"/>
@@ -1197,6 +1210,81 @@ Message is in file ${message.out}
 
   </target>
 
+  <!--
+  Create a version of the x86 tar.gz file without the aws bundle.jar.
+  If all systems had gnu tar, this would be a simple tar -delete
+  call, but bsd systems are lacking here.
+
+  Instead the workflow is
+  * untar
+  * delete
+  * retar
+  * gzip
+  -->
+  <target name="release.lean.tar" depends="init"
+    description="create a lean version of the x86 binary release">
+    <verify-release-dir />
+    <!-- files to eventually release -->
+    <setpath name="lean.tar.gz" location="${release.dir}/${release}-lean.tar.gz" />
+    <setpath name="lean.tar.gz.sha512" location="${lean.tar.gz}.sha512" />
+    <setpath name="lean.tar.gz.asc" location="${lean.tar.gz}.asc" />
+
+    <!-- intermediate files -->
+    <setpath name="lean.dir" location="target/bin-lean" />
+    <setpath name="lean.tar" location="${lean.dir}/${release}-lean.tar" />
+
+    <delete dir="${lean.dir}" />
+    <mkdir dir="${lean.dir}" />
+    <gunzip src="${release.dir}/${release}.tar.gz" dest="${lean.dir}"/>
+
+    <echo>Untarring ${lean.dir}/${release}.tar</echo>
+    <!--  use the native command to preserve properties -->
+    <x executable="tar" dir="${lean.dir}" >
+      <arg value="-xf" />
+      <arg value="${release}.tar" />
+    </x>
+    <!-- delete the bundle -->
+    <delete>
+      <fileset dir="${lean.dir}" includes="**/bundle-*.jar"/>
+    </delete>
+    <!-- retar -->
+    <echo>Creating new tar ${lean.dir}/${lean.tar}</echo>
+    <x executable="tar" dir="${lean.dir}" >
+      <arg value="-cf" />
+      <arg value="${lean.tar}" />
+      <arg value="${release}" />
+    </x>
+    <gzip src="${lean.tar}" destfile="${lean.tar.gz}"/>
+
+    <!--
+    sign it
+    -->
+    <delete file="${lean.tar.gz.sha512}" />
+    <sha512 file="${lean.tar.gz}" />
+    <loadfile srcfile="${lean.tar.gz.sha512}" property="lean.sha"/>
+    <echo>Contents of ${lean.tar.gz.sha512}
+${lean.sha}
+    </echo>
+
+    <require-file path="${lean.tar.gz}" />
+    <echo>Signing ${lean.tar.gz}</echo>
+    <delete file="${lean.tar.gz.asc}" />
+
+    <gpg dir="${release.dir}">
+      <arg value="--detach-sign" />
+      <arg value="-a" />
+      <arg value="${lean.tar.gz}" />
+    </gpg>
+    <loadfile srcfile="${lean.tar.gz.asc}" property="lean.asc"/>
+    <echo>Contents of ${lean.tar.gz.asc}
+${lean.asc}
+    </echo>
+
+    <echo>
+      Lean x86 Binary release is ${lean.tar.gz}
+    </echo>
+  </target>
+
 
   <target name="release.arm.untar" depends="release.dir.check"
     description="untar the ARM binary release">
@@ -1467,18 +1555,13 @@ gpg - -detach-sign -a  hadoop-arm64-3.3.5.tar.gz
 -->
   <target name="arm.sign.artifacts" depends="init" >
     <delete file="${arm.binary.sha512}" />
-    <checksum
-      algorithm="SHA-512"
-      fileext=".sha512"
-      file="${arm.binary}"
-      pattern="SHA512 ({1}) = {0}"
-      forceoverwrite="true"
-      />
+    <sha512 file="${arm.binary}" />
     <loadfile srcfile="${arm.binary.sha512}" property="arm.sha"/>
     <echo> contents of ${arm.binary.sha512}
 ${arm.sha}
     </echo>
 
+    <delete file="${arm.binary.asc}" />
     <gpg dir="${arm.dir}">
       <arg value="--detach-sign" />
       <arg value="-a" />


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org