You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by iw...@apache.org on 2022/05/24 05:48:32 UTC
[hadoop] branch branch-2.10.2 updated (38fd6287629 -> 227d64ab59e)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a change to branch branch-2.10.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from 38fd6287629 Preparing for 2.10.2 release
new 759c7a0557e HADOOP-15261. Upgrade commons-io from 2.4 to 2.5. Contributed by PandaMonkey.
new ba041fe6d34 YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
new 227d64ab59e YARN-11162. Set the zk acl for nodes created by ZKConfigurationStore. (#4350)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
hadoop-project/pom.xml | 2 +-
.../capacity/conf/ZKConfigurationStore.java | 12 ++++---
.../capacity/conf/TestZKConfigurationStore.java | 39 ++++++++++++++++++++++
3 files changed, 48 insertions(+), 5 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/03: HADOOP-15261. Upgrade commons-io from 2.4 to 2.5. Contributed by PandaMonkey.
Posted by iw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-2.10.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 759c7a0557e3222c0c9288070556cf44bbbe08a8
Author: Akira Ajisaka <aa...@apache.org>
AuthorDate: Fri Mar 2 15:47:43 2018 -0800
HADOOP-15261. Upgrade commons-io from 2.4 to 2.5. Contributed by PandaMonkey.
(cherry picked from commit 432cd74c7258a71eb5218dbcef0cca8eb221ddb8)
(cherry picked from commit 4803fdc4766b35a0844f7984936ab5659d93033f)
---
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a928b66c42d..d43be5a2acf 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -730,7 +730,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.4</version>
+ <version>2.5</version>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/03: YARN-11162. Set the zk acl for nodes created by ZKConfigurationStore. (#4350)
Posted by iw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-2.10.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 227d64ab59e8aa6477769b2542ad0cd7a6d855cb
Author: Owen O'Malley <oo...@linkedin.com>
AuthorDate: Mon May 23 22:07:19 2022 -0700
YARN-11162. Set the zk acl for nodes created by ZKConfigurationStore. (#4350)
(cherry picked from commit f390edaec44cfa91b2b09549091f033f1749d8ac)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
(cherry picked from commit 88a8752fa2ba0c70b0df94a78eb9fd86b965acd5)
---
.../resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 09d9e2b9f28..15c5b700879 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -97,13 +97,13 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
zkManager.delete(fencingNodePath);
if (!zkManager.exists(logsPath)) {
- zkManager.create(logsPath);
+ zkManager.create(logsPath, zkAcl);
zkManager.setData(logsPath,
serializeObject(new LinkedList<LogMutation>()), -1);
}
if (!zkManager.exists(confStorePath)) {
- zkManager.create(confStorePath);
+ zkManager.create(confStorePath, zkAcl);
HashMap<String, String> mapSchedConf = new HashMap<>();
for (Map.Entry<String, String> entry : schedConf) {
mapSchedConf.put(entry.getKey(), entry.getValue());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/03: YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
Posted by iw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-2.10.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ba041fe6d34215f075e0a7b2078d7273147e14b7
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed May 18 14:23:56 2022 +0200
YARN-11126. ZKConfigurationStore Java deserialisation vulnerability. Contributed by Tamas Domok
(cherry picked from commit 45801fba8b00257ab32c02a7d1a05948ba687a49)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
(cherry picked from commit b2be007db5bc0b731596943dbced1263a9594cde)
---
.../capacity/conf/ZKConfigurationStore.java | 8 +++--
.../capacity/conf/TestZKConfigurationStore.java | 39 ++++++++++++++++++++++
2 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 7c224a5813d..09d9e2b9f28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -19,8 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.serialization.ValidatingObjectInputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.curator.ZKCuratorManager;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -33,7 +37,6 @@ import org.apache.zookeeper.data.ACL;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.LinkedList;
@@ -229,7 +232,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
private static Object deserializeObject(byte[] bytes) throws Exception {
try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
- ObjectInputStream ois = new ObjectInputStream(bais);) {
+ ValidatingObjectInputStream ois = new ValidatingObjectInputStream(bais);) {
+ ois.accept(LinkedList.class, LogMutation.class, HashMap.class, String.class);
return ois.readObject();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 6e7cb545d30..6646bd298d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.curator.framework.CuratorFramework;
@@ -29,6 +30,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -40,9 +42,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
@@ -62,6 +66,9 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
LogFactory.getLog(TestZKConfigurationStore.class);
private static final int ZK_TIMEOUT_MS = 10000;
+ private static final String DESERIALIZATION_VULNERABILITY_FILEPATH =
+ "/tmp/ZK_DESERIALIZATION_VULNERABILITY";
+
private TestingServer curatorTestingServer;
private CuratorFramework curatorFramework;
private ResourceManager rm;
@@ -408,6 +415,38 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
rm2.close();
}
+ @Test(timeout = 3000)
+ @SuppressWarnings("checkstyle:linelength")
+ public void testDeserializationIsNotVulnerable() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ String confStorePath = ZKCuratorManager.getNodePath(
+ conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH),
+ "CONF_STORE");
+
+ File flagFile = new File(DESERIALIZATION_VULNERABILITY_FILEPATH);
+ if (flagFile.exists()) {
+ Assert.assertTrue(flagFile.delete());
+ }
+
+ // Generated using ysoserial (https://github.com/frohoff/ysoserial)
+ // java -jar ysoserial.jar CommonsBeanutils1 'touch /tmp/ZK_DESERIALIZATION_VULNERABILITY' | base64
+ ((ZKConfigurationStore) confStore).zkManager.setData(confStorePath, (new Base64(0)).decode("rO0ABXNyABdqYXZhLnV0aWwuUHJpb3JpdHlRdWV1ZZTaMLT7P4KxAwACSQAEc2l6ZUwACmNvbXBhcmF0b3J0ABZMamF2YS91dGlsL0NvbXBhcmF0b3I7eHAAAAACc3IAK29yZy5hcGFjaGUuY29tbW9ucy5iZWFudXRpbHMuQmVhbkNvbXBhcmF0b3LjoYjqcyKkSAIAAkwACmNvbXBhcmF0b3JxAH4AAUwACHByb3BlcnR5dAASTGphdmEvbGFuZy9TdHJpbmc7eHBzcgA/b3JnLmFwYWNoZS5jb21tb25zLmNvbGxlY3Rpb25zLmNvbXBhcmF0b3JzLkNvbXBhcmFibGVDb21wYXJhdG9y+/SZJbhusTcCAAB4cHQAEG91dHB1dFByb3Bl [...]
+ Assert.assertNull(confStore.retrieve());
+
+ if (!System.getProperty("os.name").startsWith("Windows")) {
+ for (int i = 0; i < 20; ++i) {
+ if (flagFile.exists()) {
+ continue;
+ }
+ Thread.sleep(100);
+ }
+
+ Assert.assertFalse("The file '" + DESERIALIZATION_VULNERABILITY_FILEPATH +
+ "' should not have been created by deserialization attack", flagFile.exists());
+ }
+ }
+
@Override
public YarnConfigurationStore createConfStore() {
return new ZKConfigurationStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org