You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2019/12/12 03:23:11 UTC
[lucene-solr] 02/02: SOLR-14053: Fix HDFS tests to be ignored if
Native I/O isn't available (remove tests.disableHdfs)
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 5b4266db7d74d8e9fc16d73b14d8306af2ec5caa
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Wed Dec 11 21:38:25 2019 -0500
SOLR-14053: Fix HDFS tests to be ignored if Native I/O isn't available (remove tests.disableHdfs)
Signed-off-by: Kevin Risden <kr...@apache.org>
---
.../TestSolrCloudWithSecureImpersonation.java | 74 +++++++++++-----------
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 23 +++++--
.../hadoop/TestDelegationWithHadoopAuth.java | 9 +--
.../hadoop/TestImpersonationWithHadoopAuth.java | 4 +-
.../hadoop/TestSolrCloudWithHadoopAuthPlugin.java | 4 +-
.../security/hadoop/TestZkAclsWithHadoopAuth.java | 26 ++------
6 files changed, 67 insertions(+), 73 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
index c8e4457..a62de35 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
@@ -24,7 +24,6 @@ import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
-import org.apache.lucene.util.Constants;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -32,8 +31,10 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.IOUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.request.SolrQueryRequest;
@@ -57,23 +58,21 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
private static String getUsersFirstGroup() throws Exception {
String group = "*"; // accept any group if a group can't be found
- if (!Constants.WINDOWS) { // does not work on Windows!
- org.apache.hadoop.security.Groups hGroups =
- new org.apache.hadoop.security.Groups(new Configuration());
- try {
- List<String> g = hGroups.getGroups(System.getProperty("user.name"));
- if (g != null && g.size() > 0) {
- group = g.get(0);
- }
- } catch (NullPointerException npe) {
- // if user/group doesn't exist on test box
+ org.apache.hadoop.security.Groups hGroups =
+ new org.apache.hadoop.security.Groups(new Configuration());
+ try {
+ List<String> g = hGroups.getGroups(System.getProperty("user.name"));
+ if (g != null && g.size() > 0) {
+ group = g.get(0);
}
+ } catch (NullPointerException npe) {
+ // if user/group doesn't exist on test box
}
return group;
}
private static Map<String, String> getImpersonatorSettings() throws Exception {
- Map<String, String> filterProps = new TreeMap<String, String>();
+ Map<String, String> filterProps = new TreeMap<>();
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noGroups.hosts", "*");
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.groups", "*");
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.hosts", "*");
@@ -93,7 +92,7 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
@BeforeClass
public static void startup() throws Exception {
- assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS);
+ HdfsTestUtil.checkAssumptions();
System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName());
System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true");
@@ -142,32 +141,35 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
}
@Before
- public void clearCalledIndicator() throws Exception {
+ public void clearCalledIndicator() {
ImpersonatorCollectionsHandler.called.set(false);
}
@AfterClass
public static void shutdown() throws Exception {
- if (miniCluster != null) {
- miniCluster.shutdown();
- }
- miniCluster = null;
if (solrClient != null) {
- solrClient.close();
- }
- solrClient = null;
- System.clearProperty("authenticationPlugin");
- System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED);
- System.clearProperty("solr.kerberos.cookie.domain");
- Map<String, String> impSettings = getImpersonatorSettings();
- for (Map.Entry<String, String> entry : impSettings.entrySet()) {
- System.clearProperty(entry.getKey());
+ IOUtils.closeQuietly(solrClient);
+ solrClient = null;
}
- System.clearProperty("solr.test.sys.prop1");
- System.clearProperty("solr.test.sys.prop2");
- System.clearProperty("collectionsHandler");
+ try {
+ if (miniCluster != null) {
+ miniCluster.shutdown();
+ }
+ } finally {
+ miniCluster = null;
+ System.clearProperty("authenticationPlugin");
+ System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED);
+ System.clearProperty("solr.kerberos.cookie.domain");
+ Map<String, String> impSettings = getImpersonatorSettings();
+ for (Map.Entry<String, String> entry : impSettings.entrySet()) {
+ System.clearProperty(entry.getKey());
+ }
+ System.clearProperty("solr.test.sys.prop1");
+ System.clearProperty("solr.test.sys.prop2");
+ System.clearProperty("collectionsHandler");
- SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false);
+ SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false);
+ }
}
private void create1ShardCollection(String name, String config, MiniSolrCloudCluster solrCluster) throws Exception {
@@ -290,8 +292,7 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
public void testProxyInvalidRemote() throws Exception {
HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
() -> {
- String invalidIpAddress = DEAD_HOST_2;
- solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", invalidIpAddress));
+ solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", DEAD_HOST_2));
});
assertTrue(e.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
}
@@ -319,15 +320,12 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
// try a command to each node, one of them must be forwarded
for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) {
- HttpSolrClient client =
- new HttpSolrClient.Builder(jetty.getBaseUrl().toString() + "/" + collectionName).build();
- try {
+ try (HttpSolrClient client = new HttpSolrClient.Builder(
+ jetty.getBaseUrl().toString() + "/" + collectionName).build()) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("q", "*:*");
params.set(USER_PARAM, "user");
client.query(params);
- } finally {
- client.close();
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
index f6f6588..c86db8b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
@@ -39,13 +39,13 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.util.HdfsUtil;
-import org.junit.AssumptionViolatedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,6 +73,20 @@ public class HdfsTestUtil {
return setupClass(dir, haTesting, true);
}
+ public static void checkAssumptions() {
+ checkHadoopWindows();
+ checkFastDateFormat();
+ checkGeneratedIdMatches();
+ }
+
+ /**
+ * Hadoop integration tests fail on Windows without Hadoop NativeIO
+ */
+ private static void checkHadoopWindows() {
+ LuceneTestCase.assumeTrue("Hadoop does not work on Windows without Hadoop NativeIO",
+ !Constants.WINDOWS || NativeIO.isAvailable());
+ }
+
/**
* Checks that commons-lang3 FastDateFormat works with configured locale
*/
@@ -96,8 +110,7 @@ public class HdfsTestUtil {
}
public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting, boolean haTesting) throws Exception {
- checkFastDateFormat();
- checkGeneratedIdMatches();
+ checkAssumptions();
if (!HA_TESTING_ENABLED) haTesting = false;
@@ -132,10 +145,6 @@ public class HdfsTestUtil {
dfsClusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology());
}
- if (!NativeIO.isAvailable()) {
- throw new AssumptionViolatedException("NativeIO not available for HDFS.");
- }
-
MiniDFSCluster dfsCluster = dfsClusterBuilder.build();
HdfsUtil.TEST_CONF = getClientConfiguration(dfsCluster);
System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
index b87b854..500f8ad 100644
--- a/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
+++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
@@ -24,7 +24,6 @@ import java.util.Set;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
-import org.apache.lucene.util.Constants;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -35,10 +34,12 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.DelegationTokenRequest;
import org.apache.solr.client.solrj.response.DelegationTokenResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.IOUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -51,7 +52,7 @@ public class TestDelegationWithHadoopAuth extends SolrCloudTestCase {
@BeforeClass
public static void setupClass() throws Exception {
- assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS);
+ HdfsTestUtil.checkAssumptions();
configureCluster(NUM_SERVERS)// nodes
.withSecurityJson(TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json"))
@@ -70,12 +71,12 @@ public class TestDelegationWithHadoopAuth extends SolrCloudTestCase {
@AfterClass
public static void tearDownClass() throws Exception {
if (primarySolrClient != null) {
- primarySolrClient.close();
+ IOUtils.closeQuietly(primarySolrClient);
primarySolrClient = null;
}
if (secondarySolrClient != null) {
- secondarySolrClient.close();
+ IOUtils.closeQuietly(secondarySolrClient);
secondarySolrClient = null;
}
}
diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java
index d603fd2..8cdcbf8 100644
--- a/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java
+++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java
@@ -23,12 +23,12 @@ import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.util.Constants;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.security.HadoopAuthPlugin;
@@ -50,7 +50,7 @@ public class TestImpersonationWithHadoopAuth extends SolrCloudTestCase {
@SuppressWarnings("unchecked")
@BeforeClass
public static void setupClass() throws Exception {
- assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS);
+ HdfsTestUtil.checkAssumptions();
InetAddress loopback = InetAddress.getLoopbackAddress();
Path securityJsonPath = TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json");
diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
index ae323a2..3043394 100644
--- a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
+++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
@@ -20,7 +20,6 @@ import java.io.File;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.FileUtils;
-import org.apache.lucene.util.Constants;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -28,6 +27,7 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.KerberosTestServices;
import org.apache.solr.cloud.SolrCloudAuthTestCase;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.SolrInputDocument;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -41,7 +41,7 @@ public class TestSolrCloudWithHadoopAuthPlugin extends SolrCloudAuthTestCase {
@BeforeClass
public static void setupClass() throws Exception {
- assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS);
+ HdfsTestUtil.checkAssumptions();
setupMiniKdc();
diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java
index 1cd0ae6..8846652 100644
--- a/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java
+++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java
@@ -27,16 +27,14 @@ import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.util.Constants;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.cloud.SecurityAwareZkACLProvider;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider;
import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.ACL;
@@ -58,7 +56,7 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase {
@BeforeClass
public static void setupClass() throws Exception {
- assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS);
+ HdfsTestUtil.checkAssumptions();
System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME,
VMParamsAllAndReadonlyDigestZkACLProvider.class.getName());
@@ -88,16 +86,9 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase {
@Test
public void testZkAcls() throws Exception {
- ZooKeeper keeper = null;
- try {
- keeper = new ZooKeeper(cluster.getZkServer().getZkAddress(), (int) TimeUnit.MINUTES.toMillis(1), new Watcher() {
- @Override
- public void process(WatchedEvent arg0) {
- // Do nothing
- }
- });
-
- keeper.addAuthInfo("digest", ("solr:"+SOLR_PASSWD).getBytes(StandardCharsets.UTF_8));
+ try (ZooKeeper keeper = new ZooKeeper(cluster.getZkServer().getZkAddress(),
+ (int) TimeUnit.MINUTES.toMillis(1), arg0 -> {/* Do nothing */})) {
+ keeper.addAuthInfo("digest", ("solr:" + SOLR_PASSWD).getBytes(StandardCharsets.UTF_8));
// Test well known paths.
checkNonSecurityACLs(keeper, "/solr.xml");
@@ -106,13 +97,8 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase {
// Now test all ZK tree.
String zkHost = cluster.getSolrClient().getZkHost();
- String zkChroot = zkHost.contains("/")? zkHost.substring(zkHost.indexOf("/")): null;
+ String zkChroot = zkHost.contains("/") ? zkHost.substring(zkHost.indexOf("/")) : null;
walkZkTree(keeper, zkChroot, "/");
-
- } finally {
- if (keeper != null) {
- keeper.close();
- }
}
}