You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2026/09/06 14:09:58 UTC
(hadoop) 02/04: phase 1 of migration
This is an automated email from the ASF dual-hosted git repository.
steveloughran pushed a commit to branch build/HADOOP-19976-reproducible
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 98b04e4babc868af5fdd56ddc54a3b69c703883a
Author: Steve Loughran <st...@apache.org>
AuthorDate: Sun Sep 6 12:58:02 2026 +0100
phase 1 of migration
repeatability
---
hadoop-common-project/hadoop-auth/pom.xml | 4 -
.../main/resources/common-version-info.properties | 4 +-
hadoop-common-project/hadoop-nfs/pom.xml | 1 -
hadoop-common-project/hadoop-registry/pom.xml | 47 ---------
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 -
.../src/main/resources/httpfs.properties | 2 +-
hadoop-maven-plugins/pom.xml | 15 +++
.../maven/plugin/versioninfo/VersionInfoMojo.java | 108 +++++++++++++++++----
.../main/resources/yarn-version-info.properties | 2 +-
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 30 ++++++
pom.xml | 46 +++++++++
11 files changed, 185 insertions(+), 76 deletions(-)
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index c35becc03a7..0822c90ac73 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -30,10 +30,6 @@
<name>Apache Hadoop Auth</name>
<description>Apache Hadoop Auth - Java HTTP SPNEGO</description>
- <properties>
- <maven.build.timestamp.format>yyyyMMdd</maven.build.timestamp.format>
- </properties>
-
<dependencies>
<dependency>
<!-- Used, even though 'mvn dependency:analyze' doesn't find it -->
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
index 0f075c8139a..4aa024e45b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
+++ b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
@@ -19,9 +19,9 @@
version=${declared.hadoop.version}
revision=${version-info.scm.commit}
branch=${version-info.scm.branch}
-user=${user.name}
+user=${hadoop.build.user}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
protocVersion=${hadoop.protobuf.version}
-compilePlatform=${os.detected.classifier}
+compilePlatform=${hadoop.build.platform}
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 685397e131d..7846a3ea1ea 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -31,7 +31,6 @@
<description>Apache Hadoop NFS library</description>
<properties>
- <maven.build.timestamp.format>yyyyMMdd</maven.build.timestamp.format>
<kerberos.realm>LOCALHOST</kerberos.realm>
</properties>
diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml
index 6a5540a17c7..3e91d4b8181 100644
--- a/hadoop-common-project/hadoop-registry/pom.xml
+++ b/hadoop-common-project/hadoop-registry/pom.xml
@@ -164,29 +164,6 @@
</dependencies>
<build>
- <!--
- Include all files in src/main/resources. By default, do not apply property
- substitution (filtering=false), but do apply property substitution to
- yarn-version-info.properties (filtering=true). This will substitute the
- version information correctly, but prevent Maven from altering other files
- like yarn-default.xml.
- -->
- <resources>
- <resource>
- <directory>${basedir}/src/main/resources</directory>
- <excludes>
- <exclude>yarn-version-info.properties</exclude>
- </excludes>
- <filtering>false</filtering>
- </resource>
- <resource>
- <directory>${basedir}/src/main/resources</directory>
- <includes>
- <include>yarn-version-info.properties</include>
- </includes>
- <filtering>true</filtering>
- </resource>
- </resources>
<plugins>
<plugin>
<groupId>com.github.spotbugs</groupId>
@@ -211,30 +188,6 @@
</excludes>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>version-info</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>version-info</goal>
- </goals>
- <configuration>
- <source>
- <directory>${basedir}/src/main</directory>
- <includes>
- <include>java/**/*.java</include>
- <!--
- <include>proto/**/*.proto</include>
- -->
- </includes>
- </source>
- </configuration>
- </execution>
- </executions>
- </plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<executions>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index f0c0e21ac12..96cf41c80ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -35,8 +35,6 @@
<properties>
<httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository>
<httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision>
- <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
- <httpfs.build.timestamp>${maven.build.timestamp}</httpfs.build.timestamp>
<kerberos.realm>LOCALHOST</kerberos.realm>
<test.exclude.kerberos.test>**/TestHttpFSWithKerberos.java</test.exclude.kerberos.test>
</properties>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs.properties b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs.properties
index 164896e1f05..9731de74898 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs.properties
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs.properties
@@ -17,5 +17,5 @@ httpfs.version=${project.version}
httpfs.source.repository=${httpfs.source.repository}
httpfs.source.revision=${httpfs.source.revision}
-httpfs.build.username=${user.name}
+httpfs.build.username=${hadoop.build.user}
httpfs.build.timestamp=${httpfs.build.timestamp}
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index e45dfa08330..d17a3e042d2 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -102,6 +102,21 @@
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.platform</groupId>
+ <artifactId>junit-platform-launcher</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
index ca970c63d01..5bd682a111b 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
@@ -29,25 +29,39 @@
import java.io.RandomAccessFile;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
-import java.util.Date;
import java.util.List;
-import java.util.TimeZone;
/**
* VersionInfoMojo calculates information about the current version of the
* codebase and exports the information as properties for further use in a Maven
* build. The version information includes build time, SCM URI, SCM branch, SCM
* commit, and an MD5 checksum of the contents of the files in the codebase.
+ * <p>
+ * The build time and the SCM fields can each be pinned to a fixed value so that
+ * two builds of the same source produce the same output; see
+ * {@code version-info.build.time.value} and {@code version-info.scm.static}.
*/
@Mojo(name="version-info")
public class VersionInfoMojo extends AbstractMojo {
+ static final String UNKNOWN = "Unknown";
+
+ /**
+ * Rendered into {@code common-version-info.properties} and read back by
+ * {@code VersionInfo}; the minute precision is long-standing and retained.
+ */
+ private static final DateTimeFormatter BUILD_TIME_FORMAT =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm'Z'")
+ .withZone(ZoneOffset.UTC);
+
@Parameter(defaultValue="${project}", readonly=true)
private MavenProject project;
@@ -72,16 +86,48 @@ public class VersionInfoMojo extends AbstractMojo {
@Parameter(defaultValue="git")
private String gitCommand;
+ /**
+ * Build time to record, as an ISO-8601 instant or as seconds since the epoch.
+ * Defaults to the Maven-wide reproducible-build timestamp; a value of one
+ * character or less means unset, following the maven-archiver convention, and
+ * the current time is used instead.
+ */
+ @Parameter(property="version-info.build.time.value",
+ defaultValue="${project.build.outputTimestamp}")
+ private String buildTimeValue;
+
+ /**
+ * When true, report the SCM values below instead of running git.
+ */
+ @Parameter(property="version-info.scm.static", defaultValue="false")
+ private boolean staticScmInfo;
+
+ @Parameter(property="version-info.scm.uri.value", defaultValue=VersionInfoMojo.UNKNOWN)
+ private String scmUriValue;
+
+ @Parameter(property="version-info.scm.branch.value", defaultValue=VersionInfoMojo.UNKNOWN)
+ private String scmBranchValue;
+
+ @Parameter(property="version-info.scm.commit.value", defaultValue=VersionInfoMojo.UNKNOWN)
+ private String scmCommitValue;
+
private enum SCM {NONE, GIT}
@Override
public void execute() throws MojoExecutionException {
try {
- SCM scm = determineSCM();
project.getProperties().setProperty(buildTimeProperty, getBuildTime());
- project.getProperties().setProperty(scmUriProperty, getSCMUri(scm));
- project.getProperties().setProperty(scmBranchProperty, getSCMBranch(scm));
- project.getProperties().setProperty(scmCommitProperty, getSCMCommit(scm));
+ if (staticScmInfo) {
+ getLog().info("SCM: static");
+ project.getProperties().setProperty(scmUriProperty, scmUriValue);
+ project.getProperties().setProperty(scmBranchProperty, scmBranchValue);
+ project.getProperties().setProperty(scmCommitProperty, scmCommitValue);
+ } else {
+ SCM scm = determineSCM();
+ project.getProperties().setProperty(scmUriProperty, getSCMUri(scm));
+ project.getProperties().setProperty(scmBranchProperty, getSCMBranch(scm));
+ project.getProperties().setProperty(scmCommitProperty, getSCMCommit(scm));
+ }
project.getProperties().setProperty(md5Property, computeMD5());
} catch (Throwable ex) {
throw new MojoExecutionException(ex.toString(), ex);
@@ -89,14 +135,40 @@ public void execute() throws MojoExecutionException {
}
/**
- * Returns a string representing current build time.
- *
- * @return String representing current build time
+ * Returns the build time to record, formatted in UTC.
+ *
+ * @return String representing the build time
+ * @throws MojoExecutionException if the configured value cannot be parsed
+ */
+ private String getBuildTime() throws MojoExecutionException {
+ Instant instant = parseBuildTime(buildTimeValue);
+ return BUILD_TIME_FORMAT.format(instant != null ? instant : Instant.now());
+ }
+
+ /**
+ * Parses a build timestamp given either as seconds since the epoch or as an
+ * ISO-8601 instant.
+ *
+ * @param value configured value; null, empty or a single character all mean
+ * "not set"
+ * @return the instant, or null if the value was not set
+ * @throws MojoExecutionException if the value is set but unparseable
*/
- private String getBuildTime() {
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- return dateFormat.format(new Date());
+ static Instant parseBuildTime(String value) throws MojoExecutionException {
+ if (value == null || value.length() <= 1) {
+ return null;
+ }
+ try {
+ return Instant.ofEpochSecond(Long.parseLong(value));
+ } catch (NumberFormatException ignored) {
+ // not epoch seconds, fall through to ISO-8601
+ }
+ try {
+ return DateTimeFormatter.ISO_DATE_TIME.parse(value, Instant::from);
+ } catch (DateTimeParseException e) {
+ throw new MojoExecutionException("Cannot parse build time '" + value
+ + "'; expected epoch seconds or an ISO-8601 instant", e);
+ }
}
private List<String> scmOut;
@@ -143,7 +215,7 @@ private SCM determineSCM() throws Exception {
* @return String URI of SCM
*/
private String getSCMUri(SCM scm) {
- String uri = "Unknown";
+ String uri = UNKNOWN;
switch (scm) {
case GIT:
for (String s : scmOut) {
@@ -165,7 +237,7 @@ private String getSCMUri(SCM scm) {
* @return String commit of SCM
*/
private String getSCMCommit(SCM scm) {
- String commit = "Unknown";
+ String commit = UNKNOWN;
switch (scm) {
case GIT:
for (String s : scmOut) {
@@ -186,7 +258,7 @@ private String getSCMCommit(SCM scm) {
* @return String branch of SCM
*/
private String getSCMBranch(SCM scm) {
- String branch = "Unknown";
+ String branch = UNKNOWN;
switch (scm) {
case GIT:
for (String s : scmOut) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
index ee6f13d77df..7178a964d33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
@@ -19,7 +19,7 @@
version=${declared.hadoop.version}
revision=${version-info.scm.commit}
branch=${version-info.scm.branch}
-user=${user.name}
+user=${hadoop.build.user}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
index c5e771b12e4..8965a809b1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml
@@ -153,6 +153,16 @@
</excludes>
</resource>
</resources>
+ <nonFilteredFileExtensions>
+ <!-- beyond the plugin's own jpg/jpeg/gif/bmp/png defaults -->
+ <nonFilteredFileExtension>ico</nonFilteredFileExtension>
+ <nonFilteredFileExtension>psd</nonFilteredFileExtension>
+ <nonFilteredFileExtension>svgz</nonFilteredFileExtension>
+ <nonFilteredFileExtension>woff</nonFilteredFileExtension>
+ <nonFilteredFileExtension>woff2</nonFilteredFileExtension>
+ <nonFilteredFileExtension>ttf</nonFilteredFileExtension>
+ <nonFilteredFileExtension>eot</nonFilteredFileExtension>
+ </nonFilteredFileExtensions>
</configuration>
</execution>
<execution>
@@ -173,6 +183,16 @@
</includes>
</resource>
</resources>
+ <nonFilteredFileExtensions>
+ <!-- beyond the plugin's own jpg/jpeg/gif/bmp/png defaults -->
+ <nonFilteredFileExtension>ico</nonFilteredFileExtension>
+ <nonFilteredFileExtension>psd</nonFilteredFileExtension>
+ <nonFilteredFileExtension>svgz</nonFilteredFileExtension>
+ <nonFilteredFileExtension>woff</nonFilteredFileExtension>
+ <nonFilteredFileExtension>woff2</nonFilteredFileExtension>
+ <nonFilteredFileExtension>ttf</nonFilteredFileExtension>
+ <nonFilteredFileExtension>eot</nonFilteredFileExtension>
+ </nonFilteredFileExtensions>
</configuration>
</execution>
<execution>
@@ -193,6 +213,16 @@
</includes>
</resource>
</resources>
+ <nonFilteredFileExtensions>
+ <!-- beyond the plugin's own jpg/jpeg/gif/bmp/png defaults -->
+ <nonFilteredFileExtension>ico</nonFilteredFileExtension>
+ <nonFilteredFileExtension>psd</nonFilteredFileExtension>
+ <nonFilteredFileExtension>svgz</nonFilteredFileExtension>
+ <nonFilteredFileExtension>woff</nonFilteredFileExtension>
+ <nonFilteredFileExtension>woff2</nonFilteredFileExtension>
+ <nonFilteredFileExtension>ttf</nonFilteredFileExtension>
+ <nonFilteredFileExtension>eot</nonFilteredFileExtension>
+ </nonFilteredFileExtensions>
</configuration>
</execution>
</executions>
diff --git a/pom.xml b/pom.xml
index ded594dee59..147935fe0a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -174,6 +174,19 @@ child.project.url.inherit.append.path="false">
release manager bumps this when cutting a release branch. -->
<hadoop.build.timestamp>2026-01-01T00:00:00Z</hadoop.build.timestamp>
+ <!-- Recorded in the version resources. Declared here rather than in the
+ module that consumes each one, so that the repeatable profile can
+ replace them: a module's own properties outrank inherited values.
+ Interpolated in the consuming module, where the os-maven-plugin
+ extension is in scope. -->
+ <hadoop.build.user>${user.name}</hadoop.build.user>
+ <hadoop.build.platform>${os.detected.classifier}</hadoop.build.platform>
+ <!-- The format belongs with the one property that consumes it. It is
+ project-wide, but httpfs is the only reference to the Maven build
+ timestamp anywhere in the tree. -->
+ <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
+ <httpfs.build.timestamp>${maven.build.timestamp}</httpfs.build.timestamp>
+
<!-- Off by default; -Prepeatable sets it to ${hadoop.build.timestamp}.
A single character disables it, per the maven-archiver convention.
Overrides the value inherited from the apache parent. -->
@@ -952,5 +965,38 @@ child.project.url.inherit.append.path="false">
<docker.image>${user.name}/hadoop:${project.version}</docker.image>
</properties>
</profile>
+
+ <!-- Replaces every value derived from the building host, user, clock or git
+ checkout with a fixed one, so that two builds of the same source with
+ the same JDK produce the same bytes. See BUILDING.txt. -->
+ <profile>
+ <id>repeatable</id>
+ <properties>
+ <!-- Picked up by maven-archiver, maven-assembly-plugin and cyclonedx to
+ normalise archive entry times and drop generation timestamps. -->
+ <project.build.outputTimestamp>${hadoop.build.timestamp}</project.build.outputTimestamp>
+
+ <!-- Recorded in the version resources in place of the git checkout, the
+ OS user and the wall clock. TestYarnVersionInfo requires a user
+ other than Unknown. -->
+ <version-info.scm.static>true</version-info.scm.static>
+ <hadoop.build.user>hadoop</hadoop.build.user>
+ <hadoop.build.platform>Unknown</hadoop.build.platform>
+ <httpfs.build.timestamp>${hadoop.build.timestamp}</httpfs.build.timestamp>
+ </properties>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <notimestamp>true</notimestamp>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
</profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org