You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by kr...@apache.org on 2022/09/21 15:11:33 UTC
[solr] 01/02: Update to Hadoop 3.3.3 (#907)
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 4e2b961508bb705c9602fe65244b606678b6c5de
Author: Mike Drob <md...@apache.org>
AuthorDate: Mon Aug 1 12:21:17 2022 -0500
Update to Hadoop 3.3.3 (#907)
* Update to Hadoop 3.3.3
* Updates to FileUtils from Hadoop upgrade
From apache/hadoop@7f33a4e99205ea8c14e1c9e50797a5dd3818fe8b
From apache/hadoop@fd96d5c2d5278aa6e7d527efa80761384c87bc26
---
solr/licenses/hadoop-annotations-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-annotations-3.3.3.jar.sha1 | 1 +
solr/licenses/hadoop-auth-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-auth-3.3.3.jar.sha1 | 1 +
solr/licenses/hadoop-client-api-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-client-api-3.3.3.jar.sha1 | 1 +
.../hadoop-client-minicluster-3.3.2.jar.sha1 | 1 -
.../hadoop-client-minicluster-3.3.3.jar.sha1 | 1 +
solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 | 1 +
solr/licenses/hadoop-common-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-common-3.3.3.jar.sha1 | 1 +
solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 | 1 -
solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 | 1 +
solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 | 1 +
solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 | 1 -
solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 | 1 +
solr/modules/hadoop-auth/build.gradle | 1 +
.../src/test/org/apache/hadoop/fs/FileUtil.java | 51 +++++++++++++++++-----
versions.lock | 18 ++++----
versions.props | 2 +-
22 files changed, 61 insertions(+), 29 deletions(-)
diff --git a/solr/licenses/hadoop-annotations-3.3.2.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.2.jar.sha1
deleted file mode 100644
index d5a4186fb51..00000000000
--- a/solr/licenses/hadoop-annotations-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2b47cd39c02c873b6bb29193962735d1d56f6572
diff --git a/solr/licenses/hadoop-annotations-3.3.3.jar.sha1 b/solr/licenses/hadoop-annotations-3.3.3.jar.sha1
new file mode 100644
index 00000000000..b5e4d34342b
--- /dev/null
+++ b/solr/licenses/hadoop-annotations-3.3.3.jar.sha1
@@ -0,0 +1 @@
+f093f02eb980be6afcda2c86aff71bcf8d9c5d9c
diff --git a/solr/licenses/hadoop-auth-3.3.2.jar.sha1 b/solr/licenses/hadoop-auth-3.3.2.jar.sha1
deleted file mode 100644
index 889352f9ca9..00000000000
--- a/solr/licenses/hadoop-auth-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-32b81a77ea6ffcbf524d1cda8ab20ea3522e6fd5
diff --git a/solr/licenses/hadoop-auth-3.3.3.jar.sha1 b/solr/licenses/hadoop-auth-3.3.3.jar.sha1
new file mode 100644
index 00000000000..4c2d0542654
--- /dev/null
+++ b/solr/licenses/hadoop-auth-3.3.3.jar.sha1
@@ -0,0 +1 @@
+6788fe94013801dca0712e3fd19b3c140f03b6e0
diff --git a/solr/licenses/hadoop-client-api-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.2.jar.sha1
deleted file mode 100644
index 9c2cc214789..00000000000
--- a/solr/licenses/hadoop-client-api-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-48f1af0a3a0270095dc59dc9f7d698969de4b4bf
diff --git a/solr/licenses/hadoop-client-api-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-api-3.3.3.jar.sha1
new file mode 100644
index 00000000000..cd914d8cd2d
--- /dev/null
+++ b/solr/licenses/hadoop-client-api-3.3.3.jar.sha1
@@ -0,0 +1 @@
+d0593aed2d4df9bcee507550913d29d589ebd84a
diff --git a/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1
deleted file mode 100644
index 379ff4a3bb3..00000000000
--- a/solr/licenses/hadoop-client-minicluster-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c08ddd065de27d21c2c2b398084092377f16a06b
diff --git a/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1
new file mode 100644
index 00000000000..d0b9ae91b1b
--- /dev/null
+++ b/solr/licenses/hadoop-client-minicluster-3.3.3.jar.sha1
@@ -0,0 +1 @@
+5820fe54634c163e08c8040747fb089f31acb90d
diff --git a/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1
deleted file mode 100644
index bd6ffdb79bc..00000000000
--- a/solr/licenses/hadoop-client-runtime-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0112f2b7420fa77c62148799175c073594197e6c
diff --git a/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1 b/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1
new file mode 100644
index 00000000000..1821815ab2a
--- /dev/null
+++ b/solr/licenses/hadoop-client-runtime-3.3.3.jar.sha1
@@ -0,0 +1 @@
+52619ecfb0225d7ae67b15264521064824ac57ca
diff --git a/solr/licenses/hadoop-common-3.3.2.jar.sha1 b/solr/licenses/hadoop-common-3.3.2.jar.sha1
deleted file mode 100644
index a97b8aa2432..00000000000
--- a/solr/licenses/hadoop-common-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-edec4cdc7f1b1208f7d135f9f228ba44b83cf58f
diff --git a/solr/licenses/hadoop-common-3.3.3.jar.sha1 b/solr/licenses/hadoop-common-3.3.3.jar.sha1
new file mode 100644
index 00000000000..46804c863ce
--- /dev/null
+++ b/solr/licenses/hadoop-common-3.3.3.jar.sha1
@@ -0,0 +1 @@
+2e1293fddedc9d5c42ce07703d22c33eb3b39aa3
diff --git a/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1
deleted file mode 100644
index a3abe8edce5..00000000000
--- a/solr/licenses/hadoop-hdfs-3.3.2-tests.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-20b47a6fe7780ae0c250081d2b9f30cd07691b6c
diff --git a/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1
deleted file mode 100644
index 7f48dfd8182..00000000000
--- a/solr/licenses/hadoop-hdfs-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aed57238fd4e669043bcc29d6bea3f0e6420950d
diff --git a/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1
new file mode 100644
index 00000000000..f8fe3f45396
--- /dev/null
+++ b/solr/licenses/hadoop-hdfs-3.3.3-tests.jar.sha1
@@ -0,0 +1 @@
+1bbe7acb60ffd432cc4bce646673279d0febdbc2
diff --git a/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1 b/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1
new file mode 100644
index 00000000000..5bc3a315c32
--- /dev/null
+++ b/solr/licenses/hadoop-hdfs-3.3.3.jar.sha1
@@ -0,0 +1 @@
+d4d199760c11d47f90e12fe3882e2b24c77e4eb5
diff --git a/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1
deleted file mode 100644
index 7e62097c3a6..00000000000
--- a/solr/licenses/hadoop-minikdc-3.3.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-749d44e9fe84566b4daa8898c62d8e88fd8e016f
diff --git a/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1 b/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1
new file mode 100644
index 00000000000..ff534b2c382
--- /dev/null
+++ b/solr/licenses/hadoop-minikdc-3.3.3.jar.sha1
@@ -0,0 +1 @@
+1871c97108af3689f82c619cfd92f01b38e4f47c
diff --git a/solr/modules/hadoop-auth/build.gradle b/solr/modules/hadoop-auth/build.gradle
index 3f614e8bc8b..47382984ef9 100644
--- a/solr/modules/hadoop-auth/build.gradle
+++ b/solr/modules/hadoop-auth/build.gradle
@@ -95,6 +95,7 @@ dependencies {
testImplementation ('org.apache.hadoop:hadoop-minikdc', {
exclude group:'org.apache.kerby', module:'kerby-xdr'
exclude group:'org.apache.kerby', module:'token-provider'
+ exclude group:'org.slf4j', module:'slf4j-reload4j'
})
// Zookeeper dependency - some tests like HdfsCloudBackupRestore need this
diff --git a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java
index dbbb2b9f29e..a960b8ed3d1 100644
--- a/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java
+++ b/solr/modules/hdfs/src/test/org/apache/hadoop/fs/FileUtil.java
@@ -54,6 +54,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
@@ -888,10 +889,13 @@ public class FileUtil {
private static void unTarUsingTar(File inFile, File untarDir,
boolean gzipped) throws IOException {
StringBuffer untarCommand = new StringBuffer();
+ // not using canonical path here; this postpones relative path
+ // resolution until bash is executed.
+ final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'";
if (gzipped) {
- untarCommand.append(" gzip -dc '")
- .append(FileUtil.makeSecureShellPath(inFile))
- .append("' | (");
+ untarCommand.append(" gzip -dc ")
+ .append(source)
+ .append(" | (");
}
untarCommand.append("cd '")
.append(FileUtil.makeSecureShellPath(untarDir))
@@ -901,15 +905,17 @@ public class FileUtil {
if (gzipped) {
untarCommand.append(" -)");
} else {
- untarCommand.append(FileUtil.makeSecureShellPath(inFile));
+ untarCommand.append(source);
}
+ LOG.debug("executing [{}]", untarCommand);
String[] shellCmd = { "bash", "-c", untarCommand.toString() };
ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException("Error untarring file " + inFile +
- ". Tar process exited with exit code " + exitcode);
+ ". Tar process exited with exit code " + exitcode
+ + " from command " + untarCommand);
}
}
@@ -966,6 +972,14 @@ public class FileUtil {
+ " would create entry outside of " + outputDir);
}
+ if (entry.isSymbolicLink() || entry.isLink()) {
+ String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
+ if (!canonicalTargetPath.startsWith(targetDirPath)) {
+ throw new IOException(
+ "expanding " + entry.getName() + " would create entry outside of " + outputDir);
+ }
+ }
+
if (entry.isDirectory()) {
File subDir = new File(outputDir, entry.getName());
if (!subDir.mkdirs() && !subDir.isDirectory()) {
@@ -981,10 +995,12 @@ public class FileUtil {
}
if (entry.isSymbolicLink()) {
- // Create symbolic link relative to tar parent dir
- Files.createSymbolicLink(FileSystems.getDefault()
- .getPath(outputDir.getPath(), entry.getName()),
- FileSystems.getDefault().getPath(entry.getLinkName()));
+ // Create symlink with canonical target path to ensure that we don't extract
+ // outside targetDirPath
+ String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
+ Files.createSymbolicLink(
+ FileSystems.getDefault().getPath(outputDir.getPath(), entry.getName()),
+ FileSystems.getDefault().getPath(canonicalTargetPath));
return;
}
@@ -996,7 +1012,8 @@ public class FileUtil {
}
if (entry.isLink()) {
- File src = new File(outputDir, entry.getLinkName());
+ String canonicalTargetPath = getCanonicalPath(entry.getLinkName(), outputDir);
+ File src = new File(canonicalTargetPath);
HardLink.createHardLink(src, outputFile);
return;
}
@@ -1004,6 +1021,20 @@ public class FileUtil {
org.apache.commons.io.FileUtils.copyToFile(tis, outputFile);
}
+ /**
+ * Gets the canonical path for the given path.
+ *
+ * @param path The path for which the canonical path needs to be computed.
+ * @param parentDir The parent directory to use if the path is a relative path.
+ * @return The canonical path of the given path.
+ */
+ private static String getCanonicalPath(String path, File parentDir) throws IOException {
+ java.nio.file.Path targetPath = Paths.get(path);
+ return (targetPath.isAbsolute() ?
+ new File(path) :
+ new File(parentDir, path)).getCanonicalPath();
+ }
+
/**
* Class for creating hardlinks.
* Supports Unix, WindXP.
diff --git a/versions.lock b/versions.lock
index 50daa2a73ff..28b35d1a9a0 100644
--- a/versions.lock
+++ b/versions.lock
@@ -132,11 +132,11 @@ org.apache.commons:commons-text:1.9 (2 constraints: 00165ad0)
org.apache.curator:curator-client:4.3.0 (2 constraints: e214cba2)
org.apache.curator:curator-framework:4.3.0 (2 constraints: ff13b474)
org.apache.curator:curator-recipes:4.3.0 (1 constraints: 09050836)
-org.apache.hadoop:hadoop-annotations:3.3.2 (1 constraints: 0a050536)
-org.apache.hadoop:hadoop-auth:3.3.2 (1 constraints: 0a050536)
-org.apache.hadoop:hadoop-client-api:3.3.2 (3 constraints: 1928ac5e)
-org.apache.hadoop:hadoop-client-runtime:3.3.2 (2 constraints: 67170443)
-org.apache.hadoop:hadoop-common:3.3.2 (1 constraints: 0a050536)
+org.apache.hadoop:hadoop-annotations:3.3.3 (1 constraints: 0b050636)
+org.apache.hadoop:hadoop-auth:3.3.3 (1 constraints: 0b050636)
+org.apache.hadoop:hadoop-client-api:3.3.3 (3 constraints: 1c28435f)
+org.apache.hadoop:hadoop-client-runtime:3.3.3 (2 constraints: 69173a43)
+org.apache.hadoop:hadoop-common:3.3.3 (1 constraints: 0b050636)
org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 (1 constraints: 0505f435)
org.apache.httpcomponents:httpclient:4.5.13 (9 constraints: 5d801b3e)
org.apache.httpcomponents:httpcore:4.4.15 (8 constraints: 1c6d2913)
@@ -275,7 +275,7 @@ org.quicktheories:quicktheories:0.26 (1 constraints: dc04f530)
org.reactivestreams:reactive-streams:1.0.3 (3 constraints: 3c2b02fd)
org.slf4j:jcl-over-slf4j:1.7.36 (3 constraints: 05188eb8)
org.slf4j:jul-to-slf4j:1.7.36 (3 constraints: 5928c263)
-org.slf4j:slf4j-api:1.7.36 (40 constraints: 02f92fa3)
+org.slf4j:slf4j-api:1.7.36 (40 constraints: 08f98db1)
org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528)
org.tallison:jmatio:1.5 (1 constraints: ff0b57e9)
org.tallison:metadata-extractor:2.17.1.0 (1 constraints: f00c3b28)
@@ -341,9 +341,9 @@ net.bytebuddy:byte-buddy:1.10.20 (2 constraints: 7c10a9d0)
net.minidev:accessors-smart:2.4.7 (1 constraints: 4e0a90b8)
net.minidev:json-smart:2.4.7 (1 constraints: 160e936e)
no.nav.security:mock-oauth2-server:0.4.3 (1 constraints: 0905fa35)
-org.apache.hadoop:hadoop-client-minicluster:3.3.2 (1 constraints: 0a050536)
-org.apache.hadoop:hadoop-hdfs:3.3.2 (1 constraints: 0a050536)
-org.apache.hadoop:hadoop-minikdc:3.3.2 (1 constraints: 0a050536)
+org.apache.hadoop:hadoop-client-minicluster:3.3.3 (1 constraints: 0b050636)
+org.apache.hadoop:hadoop-hdfs:3.3.3 (1 constraints: 0b050636)
+org.apache.hadoop:hadoop-minikdc:3.3.3 (1 constraints: 0b050636)
org.apache.kerby:kerb-admin:1.0.1 (1 constraints: 840d892f)
org.apache.kerby:kerb-client:1.0.1 (1 constraints: 840d892f)
org.apache.kerby:kerb-common:1.0.1 (2 constraints: a51841ca)
diff --git a/versions.props b/versions.props
index 23498efabca..aeb53bf0c0a 100644
--- a/versions.props
+++ b/versions.props
@@ -42,7 +42,7 @@ org.apache.commons:commons-math3=3.6.1
org.apache.commons:commons-text=1.9
org.apache.curator:*=4.3.0
org.apache.hadoop.thirdparty:*=1.1.1
-org.apache.hadoop:*=3.3.2
+org.apache.hadoop:*=3.3.3
org.apache.httpcomponents:httpclient=4.5.13
org.apache.httpcomponents:httpcore=4.4.15
org.apache.httpcomponents:httpmime=4.5.13