You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/02/01 06:40:00 UTC
hadoop git commit: HDFS-13060. Adding a
BlacklistBasedTrustedChannelResolver for TrustedChannelResolver. Contributed
by Ajay Kumar.
Repository: hadoop
Updated Branches:
refs/heads/trunk 0bee3849e -> af015c0b2
HDFS-13060. Adding a BlacklistBasedTrustedChannelResolver for TrustedChannelResolver. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af015c0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af015c0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af015c0b
Branch: refs/heads/trunk
Commit: af015c0b2359be317132e2cf35735429f4f34ea7
Parents: 0bee384
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jan 31 22:34:02 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jan 31 22:39:51 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/util/CombinedIPList.java | 59 ++++++++
.../BlackListBasedTrustedChannelResolver.java | 143 +++++++++++++++++++
.../protocol/datatransfer/package-info.java | 24 ++++
...estBlackListBasedTrustedChannelResolver.java | 89 ++++++++++++
4 files changed, 315 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af015c0b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPList.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPList.java
new file mode 100644
index 0000000..1e985e4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/CombinedIPList.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Util class to stores ips/hosts/subnets.
+ */
+public class CombinedIPList implements IPList {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(CombinedIPList.class);
+
+ private final IPList[] networkLists;
+
+ public CombinedIPList(String fixedBlackListFile,
+ String variableBlackListFile, long cacheExpiryInSeconds) {
+
+ IPList fixedNetworkList = new FileBasedIPList(fixedBlackListFile);
+ if (variableBlackListFile != null) {
+ IPList variableNetworkList = new CacheableIPList(
+ new FileBasedIPList(variableBlackListFile), cacheExpiryInSeconds);
+ networkLists = new IPList[]{fixedNetworkList, variableNetworkList};
+ } else {
+ networkLists = new IPList[]{fixedNetworkList};
+ }
+ }
+
+ @Override
+ public boolean isIn(String ipAddress) {
+ if (ipAddress == null) {
+ throw new IllegalArgumentException("ipAddress is null");
+ }
+
+ for (IPList networkList : networkLists) {
+ if (networkList.isIn(ipAddress)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af015c0b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlackListBasedTrustedChannelResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlackListBasedTrustedChannelResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlackListBasedTrustedChannelResolver.java
new file mode 100644
index 0000000..4fb2416
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlackListBasedTrustedChannelResolver.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol.datatransfer;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.CombinedIPList;
+
+/**
+ * Implements {@link TrustedChannelResolver}
+ * to trust ips/host/subnets based on a blackList.
+ */
+public class BlackListBasedTrustedChannelResolver extends
+ TrustedChannelResolver {
+
+ private CombinedIPList blackListForServer;
+ private CombinedIPList blackListForClient;
+
+ private static final String FIXED_BLACK_LIST_DEFAULT_LOCATION = "/etc/hadoop"
+ + "/fixedBlackList";
+
+ private static final String VARIABLE_BLACK_LIST_DEFAULT_LOCATION = "/etc/"
+ + "hadoop/blackList";
+
+ /**
+ * Path to the file containing subnets and ip addresses to form
+ * fixed BlackList. Server side config.
+ */
+ public static final String DFS_DATATRANSFER_SERVER_FIXED_BLACK_LIST_FILE =
+ "dfs.datatransfer.server.fixedBlackList.file";
+ /**
+ * Enables/Disables variable BlackList. Server side config.
+ */
+ public static final String DFS_DATATRANSFER_SERVER_VARIABLE_BLACK_LIST_ENABLE
+ = "dfs.datatransfer.server.variableBlackList.enable";
+ /**
+ * Path to the file containing subnets and ip addresses to form
+ * variable BlackList. Server side config.
+ */
+ public static final String DFS_DATATRANSFER_SERVER_VARIABLE_BLACK_LIST_FILE =
+ "dfs.datatransfer.server.variableBlackList.file";
+ /**
+ * Time in seconds after which the variable BlackList file is checked for
+ * updates. Server side config.
+ */
+ public static final String
+ DFS_DATATRANSFER_SERVER_VARIABLE_BLACK_LIST_CACHE_SECS = "dfs."
+ + "datatransfer.server.variableBlackList.cache.secs";
+
+ /**
+ * Path to the file containing subnets and ip addresses to
+ * form fixed BlackList. This key is for client.
+ */
+ public static final String DFS_DATATRANSFER_CLIENT_FIXED_BLACK_LIST_FILE =
+ "dfs.datatransfer.client.fixedBlackList.file";
+ /**
+ * Enables/Disables variable BlackList. This key is for client.
+ */
+ public static final String DFS_DATATRANSFER_CLIENT_VARIABLE_BLACK_LIST_ENABLE
+ = "dfs.datatransfer.client.variableBlackList.enable";
+ /**
+ * Path to the file to containing subnets and ip addresses to form variable
+ * BlackList. This key is for client.
+ */
+ public static final String DFS_DATATRANSFER_CLIENT_VARIABLE_BLACK_LIST_FILE =
+ "dfs.datatransfer.client.variableBlackList.file";
+ /**
+ * Time in seconds after which the variable BlackList file is
+ * checked for updates. This key is for client.
+ */
+ public static final String
+ DFS_DATATRANSFER_CLIENT_VARIABLE_BLACK_LIST_CACHE_SECS =
+ "dfs.datatransfer.client.variableBlackList.cache.secs";
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ String fixedFile = conf.get(DFS_DATATRANSFER_SERVER_FIXED_BLACK_LIST_FILE,
+ FIXED_BLACK_LIST_DEFAULT_LOCATION);
+ String variableFile = null;
+ long expiryTime = 0;
+
+ if (conf
+ .getBoolean(DFS_DATATRANSFER_SERVER_VARIABLE_BLACK_LIST_ENABLE,
+ false)) {
+ variableFile = conf.get(DFS_DATATRANSFER_SERVER_VARIABLE_BLACK_LIST_FILE,
+ VARIABLE_BLACK_LIST_DEFAULT_LOCATION);
+ expiryTime =
+ conf.getLong(DFS_DATATRANSFER_SERVER_VARIABLE_BLACK_LIST_CACHE_SECS,
+ 3600) * 1000;
+ }
+
+ blackListForServer = new CombinedIPList(fixedFile, variableFile,
+ expiryTime);
+
+ fixedFile = conf
+ .get(DFS_DATATRANSFER_CLIENT_FIXED_BLACK_LIST_FILE, fixedFile);
+ expiryTime = 0;
+
+ if (conf
+ .getBoolean(DFS_DATATRANSFER_CLIENT_VARIABLE_BLACK_LIST_ENABLE,
+ false)) {
+ variableFile = conf
+ .get(DFS_DATATRANSFER_CLIENT_VARIABLE_BLACK_LIST_FILE, variableFile);
+ expiryTime =
+ conf.getLong(DFS_DATATRANSFER_CLIENT_VARIABLE_BLACK_LIST_CACHE_SECS,
+ 3600) * 1000;
+ }
+
+ blackListForClient = new CombinedIPList(fixedFile, variableFile,
+ expiryTime);
+ }
+
+ public boolean isTrusted() {
+ try {
+ return !blackListForClient
+ .isIn(InetAddress.getLocalHost().getHostAddress());
+ } catch (UnknownHostException e) {
+ return true;
+ }
+ }
+
+ public boolean isTrusted(InetAddress clientAddress) {
+ return !blackListForServer.isIn(clientAddress.getHostAddress());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af015c0b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
new file mode 100644
index 0000000..a13c7d8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceStability.Evolving
+package org.apache.hadoop.hdfs.protocol.datatransfer;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This package contains classes related to hdfs data transfer protocol.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af015c0b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestBlackListBasedTrustedChannelResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestBlackListBasedTrustedChannelResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestBlackListBasedTrustedChannelResolver.java
new file mode 100644
index 0000000..75e1a4d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestBlackListBasedTrustedChannelResolver.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol.datatransfer.sasl;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.protocol.datatransfer.BlackListBasedTrustedChannelResolver;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link BlackListBasedTrustedChannelResolver}.
+ */
+public class TestBlackListBasedTrustedChannelResolver {
+
+ private final static String FILE_NAME = "blacklistfile.txt";
+ private File blacklistFile;
+ private final static String BLACK_LISTED = "127.0.0.1\n216.58.216.174\n";
+ private BlackListBasedTrustedChannelResolver resolver;
+
+ @Before
+ public void setup() {
+ blacklistFile = new File(GenericTestUtils.getTestDir(), FILE_NAME);
+ resolver
+ = new BlackListBasedTrustedChannelResolver();
+ try {
+ FileUtils.write(blacklistFile, BLACK_LISTED);
+ } catch (IOException e) {
+ fail("Setup for TestBlackListBasedTrustedChannelResolver failed.");
+ }
+ }
+
+ @After
+ public void cleanUp() {
+ FileUtils.deleteQuietly(blacklistFile);
+ }
+
+ @Test
+ public void testBlackListIpClient() throws IOException {
+ Configuration conf = new Configuration();
+ FileUtils.write(blacklistFile,
+ InetAddress.getLocalHost().getHostAddress(), true);
+ conf.set(BlackListBasedTrustedChannelResolver
+ .DFS_DATATRANSFER_CLIENT_FIXED_BLACK_LIST_FILE,
+ blacklistFile.getAbsolutePath());
+
+ resolver.setConf(conf);
+ assertFalse(resolver.isTrusted());
+
+ }
+
+ @Test
+ public void testBlackListIpServer() throws UnknownHostException {
+ Configuration conf = new Configuration();
+ conf.set(BlackListBasedTrustedChannelResolver
+ .DFS_DATATRANSFER_SERVER_FIXED_BLACK_LIST_FILE,
+ blacklistFile.getAbsolutePath());
+
+ resolver.setConf(conf);
+ assertTrue(resolver.isTrusted());
+ assertFalse(resolver.isTrusted(InetAddress
+ .getByName("216.58.216.174")));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org