You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2019/03/18 19:26:17 UTC
[lucene-solr] 01/01: SOLR-13330: Improve HDFS tests
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch jira/solr-13330
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit cf828163bdfa010c87f1171b6919e444bd0ff01c
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Tue Mar 12 13:06:19 2019 -0400
SOLR-13330: Improve HDFS tests
Related JIRAs:
* SOLR-11010
* SOLR-11381
* SOLR-12040
* SOLR-13297
Changes:
* Consolidate hdfs configuration into HdfsTestUtil
* Ensure socketTimeout long enough for HDFS tests
* Ensure HdfsTestUtil.getClientConfiguration used in tests
* Replace deprecated HDFS calls
* Use try-with-resources to ensure closing of HDFS resources
Signed-off-by: Kevin Risden <kr...@apache.org>
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/core/HdfsDirectoryFactory.java | 2 +-
.../solr/collection1/conf/solrconfig-minhash.xml | 4 +-
.../solr/collection1/conf/solrconfig.xml | 4 +-
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 5 -
.../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 4 -
.../apache/solr/cloud/UnloadDistributedZkTest.java | 38 +++-----
.../HdfsCollectionsAPIDistributedZkTest.java | 4 -
.../collections/TestHdfsCloudBackupRestore.java | 1 -
.../HdfsAutoAddReplicasIntegrationTest.java | 6 --
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 5 -
.../cloud/hdfs/HdfsBasicDistributedZkTest.java | 2 -
.../solr/cloud/hdfs/HdfsRecoverLeaseTest.java | 6 +-
.../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 16 +---
.../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 10 +-
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 104 ++++++++++++---------
.../apache/solr/cloud/hdfs/HdfsThreadLeakTest.java | 43 +++------
.../HdfsTlogReplayBufferedWhileIndexingTest.java | 6 +-
.../cloud/hdfs/HdfsUnloadDistributedZkTest.java | 4 +-
.../hdfs/HdfsWriteToMultipleCollectionsTest.java | 7 +-
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 1 -
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 68 ++++++++------
.../solr/handler/TestHdfsBackupRestoreCore.java | 22 +++--
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 24 +++--
.../org/apache/solr/search/TestRecoveryHdfs.java | 62 ++++++------
.../org/apache/solr/update/TestHdfsUpdateLog.java | 7 +-
26 files changed, 206 insertions(+), 251 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1c6ee04..bdb7350 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -158,6 +158,8 @@ Other Changes
* SOLR-13307: Ensure HDFS tests clear System properties they set (Kevin Risden)
+* SOLR-13330: Improve HDFS tests (Kevin Risden)
+
================== 8.0.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index 13e1de1..a0806ed 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -531,7 +531,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
boolean accept = false;
String pathName = path.getName();
try {
- accept = fs.isDirectory(path) && !path.equals(currentIndexDirPath) &&
+ accept = fs.getFileStatus(path).isDirectory() && !path.equals(currentIndexDirPath) &&
(pathName.equals("index") || pathName.matches(INDEX_W_TIMESTAMP_REGEX));
} catch (IOException e) {
log.error("Error checking if path {} is an old index directory, caused by: {}", path, e);
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
index 333c1db..3df6f2b 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml
@@ -49,8 +49,8 @@
<bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
<bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
<bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">10</int>
- <int name="solr.hdfs.blockcache.slab.count">1</int>
+ <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
+ <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
index 9e7e505..c68e7e0 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
@@ -49,8 +49,8 @@
<bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
<bool name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</bool>
<bool name="solr.hdfs.blockcache.write.enabled">${solr.hdfs.blockcache.write.enabled:false}</bool>
- <int name="solr.hdfs.blockcache.blocksperbank">10</int>
- <int name="solr.hdfs.blockcache.slab.count">1</int>
+ <int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:10}</int>
+ <int name="solr.hdfs.blockcache.slab.count">${solr.hdfs.blockcache.slab.count:1}</int>
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
index 2a26956..f66029b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
@@ -52,13 +52,10 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
.configure();
- System.setProperty("solr.hdfs.blockcache.enabled", "false");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
ZkConfigManager configManager = new ZkConfigManager(zkClient());
configManager.uploadConfigDir(configset("cloud-hdfs"), "conf1");
-
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getDataDir(dfsCluster, "data"));
}
@AfterClass
@@ -70,8 +67,6 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.blockcache.enabled");
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index 9678c2e..8204027 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -39,8 +39,6 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
- System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -50,8 +48,6 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
- System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
index 9437748..003e933 100644
--- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
@@ -31,7 +31,6 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.SolrCore;
@@ -42,6 +41,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Collection;
+import java.util.Collections;
import java.util.Random;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -54,23 +54,19 @@ import java.util.concurrent.TimeUnit;
@Slow
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class UnloadDistributedZkTest extends BasicDistributedZkTest {
+ public UnloadDistributedZkTest() {
+ super();
+ }
protected String getSolrXml() {
return "solr.xml";
}
-
- public UnloadDistributedZkTest() {
- super();
- }
@Test
- //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018
public void test() throws Exception {
-
testCoreUnloadAndLeaders(); // long
testUnloadLotsOfCores(); // long
-
+
testUnloadShardAndCollection();
}
@@ -78,9 +74,9 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
boolean shouldBePresent, int expectedSliceCount) throws Exception {
final TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME);
Boolean isPresent = null; // null meaning "don't know"
- while (null == isPresent || shouldBePresent != isPresent.booleanValue()) {
+ while (null == isPresent || shouldBePresent != isPresent) {
final DocCollection docCollection = getCommonCloudSolrClient().getZkStateReader().getClusterState().getCollectionOrNull(collectionName);
- final Collection<Slice> slices = (docCollection != null) ? docCollection.getSlices() : null;
+ final Collection<Slice> slices = (docCollection != null) ? docCollection.getSlices() : Collections.emptyList();
if (timeout.hasTimedOut()) {
printLayout();
fail("checkCoreNamePresenceAndSliceCount failed:"
@@ -88,14 +84,12 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
+" shouldBePresent="+shouldBePresent+" isPresent="+isPresent
+" expectedSliceCount="+expectedSliceCount+" actualSliceCount="+slices.size());
}
- if (expectedSliceCount == (slices == null ? 0 : slices.size())) {
+ if (expectedSliceCount == slices.size()) {
isPresent = false;
- if (slices != null) {
- for (Slice slice : slices) {
- for (Replica replica : slice.getReplicas()) {
- if (coreName.equals(replica.get("core"))) {
- isPresent = true;
- }
+ for (Slice slice : slices) {
+ for (Replica replica : slice.getReplicas()) {
+ if (coreName.equals(replica.get("core"))) {
+ isPresent = true;
}
}
}
@@ -105,7 +99,6 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
}
private void testUnloadShardAndCollection() throws Exception{
-
final int numShards = 2;
final String collection = "test_unload_shard_and_collection";
@@ -285,7 +278,6 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
Unload unloadCmd = new Unload(false);
unloadCmd.setCoreName(leaderProps.getCoreName());
- SolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
collectionClient.request(unloadCmd);
}
tries = 50;
@@ -335,7 +327,6 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
assertEquals(found1, found3);
assertEquals(found3, found4);
}
-
}
private void testUnloadLotsOfCores() throws Exception {
@@ -343,7 +334,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
try (final HttpSolrClient adminClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {
int numReplicas = atLeast(3);
ThreadPoolExecutor executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE,
- 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ 5, TimeUnit.SECONDS, new SynchronousQueue<>(),
new DefaultSolrThreadFactory("testExecutor"));
try {
// create the cores
@@ -353,7 +344,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
}
executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, 5,
- TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ TimeUnit.SECONDS, new SynchronousQueue<>(),
new DefaultSolrThreadFactory("testExecutor"));
try {
for (int j = 0; j < numReplicas; j++) {
@@ -374,5 +365,4 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
}
}
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
index 700165b..94b3102 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
@@ -38,8 +38,6 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
- System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -49,8 +47,6 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
- System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index a9901e8..0d876db 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -112,7 +112,6 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
if (fs instanceof DistributedFileSystem) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
index 37ce805..8d8ef9c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
@@ -38,9 +38,6 @@ public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrati
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.global", "true");
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
- System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -50,9 +47,6 @@ public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrati
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.global");
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
- System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
index c06b28a..6a93042 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
@@ -43,13 +43,10 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {
configureCluster(2)
.configure();
- System.setProperty("solr.hdfs.blockcache.enabled", "false");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
ZkConfigManager configManager = new ZkConfigManager(zkClient());
configManager.uploadConfigDir(configset("cloud-hdfs"), "conf1");
-
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getDataDir(dfsCluster, "data"));
}
@@ -62,8 +59,6 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.enabled");
- System.clearProperty("solr.hdfs.home");
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
index 59293c6..192e7ee 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
@@ -38,7 +38,6 @@ public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest {
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -53,7 +52,6 @@ public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
index fcacb8b..ad8556c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
@@ -72,8 +72,7 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
URI uri = dfsCluster.getURI();
Path path = new Path(uri);
- Configuration conf = new Configuration();
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
FileSystem fs1 = FileSystem.get(path.toUri(), conf);
Path testFile = new Path(uri.toString() + "/testfile");
FSDataOutputStream out = fs1.create(testFile);
@@ -131,8 +130,7 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
final URI uri = dfsCluster.getURI();
final Path path = new Path(uri);
- final Configuration conf = new Configuration();
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ final Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
// n threads create files
class WriterThread extends Thread {
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
index b881292..6e6bd3d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
@@ -18,35 +18,30 @@ package org.apache.solr.cloud.hdfs;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.lucene.util.LuceneTestCase.Nightly;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.RecoveryZkTest;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.util.BadHdfsThreadsFilter;
-import org.apache.solr.util.LogLevel;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@Slow
-@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
-//@Nightly
+@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+@Nightly
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-@LogLevel("org.apache.solr.update.HdfsTransactionLog=DEBUG")
public class HdfsRecoveryZkTest extends RecoveryZkTest {
-
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
ZkConfigManager configManager = new ZkConfigManager(zkClient());
configManager.uploadConfigDir(configset("cloud-hdfs"), "conf");
-
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getDataDir(dfsCluster, "data"));
}
@AfterClass
@@ -58,10 +53,7 @@ public class HdfsRecoveryZkTest extends RecoveryZkTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
- System.clearProperty("solr.hdfs.home");
}
}
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
index 02a7943..64af109 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
@@ -16,8 +16,6 @@
*/
package org.apache.solr.cloud.hdfs;
-import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.RestartWhileUpdatingTest;
@@ -34,16 +32,15 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest {
+ private static MiniDFSCluster dfsCluster;
+
public HdfsRestartWhileUpdatingTest() throws Exception {
super();
}
- private static MiniDFSCluster dfsCluster;
-
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
}
@AfterClass
@@ -52,12 +49,11 @@ public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
}
}
@Override
- protected String getDataDir(String dataDir) throws IOException {
+ protected String getDataDir(String dataDir) {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
index d601565..6987a6a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
@@ -17,9 +17,9 @@
package org.apache.solr.cloud.hdfs;
import java.io.File;
-import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
+import java.util.Enumeration;
import java.util.Locale;
import java.util.Map;
import java.util.Timer;
@@ -47,6 +47,8 @@ import org.apache.solr.util.HdfsUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.lucene.util.LuceneTestCase.random;
+
public class HdfsTestUtil {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -99,49 +101,49 @@ public class HdfsTestUtil {
if (!HA_TESTING_ENABLED) haTesting = false;
- int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 2);
-
- Configuration conf = new Configuration();
- conf.set("dfs.block.access.token.enable", "false");
- conf.set("dfs.permissions.enabled", "false");
- conf.set("hadoop.security.authentication", "simple");
+ Configuration conf = getBasicConfiguration(new Configuration());
conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir");
conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir");
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ // Disable metrics logging for HDFS
+ conf.setInt("dfs.namenode.metrics.logger.period.seconds", 0);
+ conf.setInt("dfs.datanode.metrics.logger.period.seconds", 0);
System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build");
System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
- System.setProperty("solr.hdfs.blockcache.global",
- System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean())));
-
- final MiniDFSCluster dfsCluster;
-
- if (!haTesting) {
- dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes).format(true).build();
+ // test-files/solr/solr.xml sets this to be 15000. This isn't long enough for HDFS in some cases.
+ System.setProperty("socketTimeout", "90000");
- System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
+ String blockcacheGlobal = System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(random().nextBoolean()));
+ System.setProperty("solr.hdfs.blockcache.global", blockcacheGlobal);
+ // Limit memory usage for HDFS tests
+ if(Boolean.parseBoolean(blockcacheGlobal)) {
+ System.setProperty("solr.hdfs.blockcache.blocksperbank", "4096");
} else {
- dfsCluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(dataNodes)
- .build();
-
- Configuration haConfig = getClientConfiguration(dfsCluster);
+ System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
+ System.setProperty("tests.hdfs.numdatanodes", "1");
+ }
- HdfsUtil.TEST_CONF = haConfig;
- System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
+ int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 2);
+ final MiniDFSCluster.Builder dfsClusterBuilder = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(dataNodes).format(true);
+ if (haTesting) {
+ dfsClusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology());
}
+ MiniDFSCluster dfsCluster = dfsClusterBuilder.build();
+ HdfsUtil.TEST_CONF = getClientConfiguration(dfsCluster);
+ System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
dfsCluster.waitActive();
if (haTesting) dfsCluster.transitionToActive(0);
- int rndMode = LuceneTestCase.random().nextInt(3);
+ int rndMode = random().nextInt(3);
if (safeModeTesting && rndMode == 1) {
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
- int rnd = LuceneTestCase.random().nextInt(10000);
+ int rnd = random().nextInt(10000);
Timer timer = new Timer();
timers.put(dfsCluster, timer);
timer.schedule(new TimerTask() {
@@ -151,9 +153,8 @@ public class HdfsTestUtil {
NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
}
}, rnd);
-
} else if (haTesting && rndMode == 2) {
- int rnd = LuceneTestCase.random().nextInt(30000);
+ int rnd = random().nextInt(30000);
Timer timer = new Timer();
timers.put(dfsCluster, timer);
timer.schedule(new TimerTask() {
@@ -176,7 +177,7 @@ public class HdfsTestUtil {
URI uri = dfsCluster.getURI();
Path hdfsDirPath = new Path(uri.toString() + "/solr/collection1/core_node1/data/tlog/tlog.0000000000000000000");
// tran log already being created testing
- badTlogOutStreamFs = FileSystem.get(hdfsDirPath.toUri(), conf);
+ badTlogOutStreamFs = FileSystem.get(hdfsDirPath.toUri(), getClientConfiguration(dfsCluster));
badTlogOutStream = badTlogOutStreamFs.create(hdfsDirPath);
}
@@ -185,18 +186,23 @@ public class HdfsTestUtil {
return dfsCluster;
}
+ private static Configuration getBasicConfiguration(Configuration conf) {
+ conf.setBoolean("dfs.block.access.token.enable", false);
+ conf.setBoolean("dfs.permissions.enabled", false);
+ conf.set("hadoop.security.authentication", "simple");
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ return conf;
+ }
+
public static Configuration getClientConfiguration(MiniDFSCluster dfsCluster) {
+ Configuration conf = getBasicConfiguration(dfsCluster.getConfiguration(0));
if (dfsCluster.getNameNodeInfos().length > 1) {
- Configuration conf = new Configuration();
HATestUtil.setFailoverConfigurations(dfsCluster, conf);
- return conf;
- } else {
- return new Configuration();
}
+ return conf;
}
public static void teardownClass(MiniDFSCluster dfsCluster) throws Exception {
-
if (badTlogOutStream != null) {
IOUtils.closeQuietly(badTlogOutStream);
}
@@ -205,16 +211,19 @@ public class HdfsTestUtil {
IOUtils.closeQuietly(badTlogOutStreamFs);
}
- SolrTestCaseJ4.resetFactory();
-
try {
+ try {
+ SolrTestCaseJ4.resetFactory();
+ } catch (Exception e) {
+ log.error("Exception trying to reset solr.directoryFactory", e);
+ }
if (dfsCluster != null) {
Timer timer = timers.remove(dfsCluster);
if (timer != null) {
timer.cancel();
}
try {
- dfsCluster.shutdown();
+ dfsCluster.shutdown(true);
} catch (Error e) {
// Added in SOLR-7134
// Rarely, this can fail to either a NullPointerException
@@ -224,16 +233,27 @@ public class HdfsTestUtil {
}
}
} finally {
- System.clearProperty("solr.lock.type");
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.blockcache.global");
+
+ System.clearProperty("socketTimeout");
+
+ System.clearProperty("tests.hdfs.numdatanodes");
+
+ System.clearProperty("solr.lock.type");
+
+ // Clear "solr.hdfs." system properties
+ Enumeration<?> propertyNames = System.getProperties().propertyNames();
+ while(propertyNames.hasMoreElements()) {
+ String propertyName = String.valueOf(propertyNames.nextElement());
+ if(propertyName.startsWith("solr.hdfs.")) {
+ System.clearProperty(propertyName);
+ }
+ }
}
}
- public static String getDataDir(MiniDFSCluster dfsCluster, String dataDir)
- throws IOException {
+ public static String getDataDir(MiniDFSCluster dfsCluster, String dataDir) {
if (dataDir == null) {
return null;
}
@@ -250,7 +270,7 @@ public class HdfsTestUtil {
return "hdfs://" + logicalName;
} else {
URI uri = dfsCluster.getURI(0);
- return uri.toString() ;
+ return uri.toString();
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
index 1cd6819..f250d53 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
@@ -25,9 +25,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.BadHdfsThreadsFilter;
-import org.junit.After;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,7 +35,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
-
private static MiniDFSCluster dfsCluster;
@BeforeClass
@@ -47,38 +44,26 @@ public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
@AfterClass
public static void afterClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
- }
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
- }
-
- @After
- public void tearDown() throws Exception {
- super.tearDown();
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Test
public void testBasic() throws IOException {
String uri = HdfsTestUtil.getURI(dfsCluster);
Path path = new Path(uri);
- Configuration conf = new Configuration();
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
- FileSystem fs = FileSystem.get(path.toUri(), conf);
- Path testFile = new Path(uri.toString() + "/testfile");
- FSDataOutputStream out = fs.create(testFile);
-
- out.write(5);
- out.hflush();
- out.close();
-
- ((DistributedFileSystem) fs).recoverLease(testFile);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ try(FileSystem fs = FileSystem.get(path.toUri(), conf)) {
+ Path testFile = new Path(uri + "/testfile");
+ try(FSDataOutputStream out = fs.create(testFile)) {
+ out.write(5);
+ out.hflush();
+ }
-
- fs.close();
+ ((DistributedFileSystem) fs).recoverLease(testFile);
+ }
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
index 52b3a82..ff19b09 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
@@ -16,8 +16,6 @@
*/
package org.apache.solr.cloud.hdfs;
-import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest;
@@ -42,7 +40,6 @@ public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedW
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -52,12 +49,11 @@ public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedW
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
}
}
@Override
- protected String getDataDir(String dataDir) throws IOException {
+ protected String getDataDir(String dataDir) {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
index c15b8c3..0d3d23e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
@@ -16,8 +16,6 @@
*/
package org.apache.solr.cloud.hdfs;
-import java.io.IOException;
-
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.UnloadDistributedZkTest;
@@ -51,7 +49,7 @@ public class HdfsUnloadDistributedZkTest extends UnloadDistributedZkTest {
}
@Override
- protected String getDataDir(String dataDir) throws IOException {
+ protected String getDataDir(String dataDir) {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
index 6f48b98..1f31603 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
@@ -141,10 +141,9 @@ public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest {
Directory dir = factory.get(core.getDataDir(), null, null);
try {
long dataDirSize = factory.size(dir);
- FileSystem fileSystem = null;
-
- fileSystem = FileSystem.newInstance(
- new Path(core.getDataDir()).toUri(), new Configuration());
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ FileSystem fileSystem = FileSystem.newInstance(
+ new Path(core.getDataDir()).toUri(), conf);
long size = fileSystem.getContentSummary(
new Path(core.getDataDir())).getLength();
assertEquals(size, dataDirSize);
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
index b3d5e27..04f9387 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
@@ -229,7 +229,6 @@ public class StressHdfsTest extends BasicDistributedZkTest {
// check that all dirs are gone
for (String dataDir : dataDirs) {
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
try(FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) {
assertFalse(
"Data directory exists after collection removal : " + dataDir,
diff --git a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
index c599fb7..e8c16a3 100644
--- a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.core;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -24,7 +26,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.Random;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.store.Directory;
@@ -55,7 +56,6 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "1024");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
}
@@ -65,41 +65,42 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
+ System.clearProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY);
+ System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ System.clearProperty(HdfsDirectoryFactory.LOCALITYMETRICS_ENABLED);
}
}
@Test
public void testInitArgsOrSysPropConfig() throws Exception {
try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
-
// test sys prop config
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr1/mock/data"));
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
// test init args config
NamedList<Object> nl = new NamedList<>();
- nl.add("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr2");
+ nl.add(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr2");
hdfsFactory.init(nl);
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr2/mock/data"));
// test sys prop and init args config - init args wins
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(nl);
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr2/mock/data"));
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
// set conf dir by sys prop
Path confDir = createTempDir();
@@ -141,44 +142,49 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@Test
public void testCleanupOldIndexDirectories() throws Exception {
try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr1/mock/data"));
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
- FileSystem hdfs = dfsCluster.getFileSystem();
+ try(FileSystem hdfs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
+ org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome);
+ org.apache.hadoop.fs.Path currentIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index");
+ assertFalse(checkHdfsDirectory(hdfs,currentIndexDirPath));
+ hdfs.mkdirs(currentIndexDirPath);
+ assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath));
- org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome);
- org.apache.hadoop.fs.Path currentIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index");
- assertTrue(!hdfs.isDirectory(currentIndexDirPath));
- hdfs.mkdirs(currentIndexDirPath);
- assertTrue(hdfs.isDirectory(currentIndexDirPath));
+ String timestamp1 = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
+ org.apache.hadoop.fs.Path oldIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1);
+ assertFalse(checkHdfsDirectory(hdfs,oldIndexDirPath));
+ hdfs.mkdirs(oldIndexDirPath);
+ assertTrue(checkHdfsDirectory(hdfs, oldIndexDirPath));
- String timestamp1 = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date());
- org.apache.hadoop.fs.Path oldIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1);
- assertTrue(!hdfs.isDirectory(oldIndexDirPath));
- hdfs.mkdirs(oldIndexDirPath);
- assertTrue(hdfs.isDirectory(oldIndexDirPath));
+ hdfsFactory.cleanupOldIndexDirectories(dataHomePath.toString(), currentIndexDirPath.toString(), false);
- hdfsFactory.cleanupOldIndexDirectories(dataHomePath.toString(), currentIndexDirPath.toString(), false);
+ assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath));
+ assertFalse(checkHdfsDirectory(hdfs, oldIndexDirPath));
+ }
+ }
+ }
- assertTrue(hdfs.isDirectory(currentIndexDirPath));
- assertTrue(!hdfs.isDirectory(oldIndexDirPath));
+ private boolean checkHdfsDirectory(FileSystem hdfs, org.apache.hadoop.fs.Path path) throws IOException {
+ try {
+ return hdfs.getFileStatus(path).isDirectory();
+ } catch (FileNotFoundException e) {
+ return false;
}
}
@Test
public void testLocalityReporter() throws Exception {
- Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.set("dfs.permissions.enabled", "false");
-
Random r = random();
try(HdfsDirectoryFactory factory = new HdfsDirectoryFactory()) {
SolrMetricManager metricManager = new SolrMetricManager();
String registry = TestUtil.randomSimpleString(r, 2, 10);
String scope = TestUtil.randomSimpleString(r, 2, 10);
- Map<String, String> props = new HashMap<String, String>();
+ Map<String, String> props = new HashMap<>();
props.put(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr");
props.put(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
props.put(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_ENABLE, "false");
@@ -190,7 +196,7 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
MetricsMap metrics = (MetricsMap) ((SolrMetricManager.GaugeWrapper) metricManager.registry(registry).getMetrics().get("OTHER." + scope + ".hdfsLocality")).getGauge();
// We haven't done anything, so there should be no data
Map<String, Object> statistics = metrics.getValue();
- assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0l,
+ assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0L,
statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
assertEquals(
"Counted bytes as local when none written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO), 0,
diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
index 60f77b4..f51c4ac 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -109,7 +108,6 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
if (fs instanceof DistributedFileSystem) {
@@ -147,13 +145,17 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
IOUtils.closeQuietly(fs);
fs = null;
try {
- HdfsTestUtil.teardownClass(dfsCluster);
+ SolrTestCaseJ4.resetFactory();
} finally {
- dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
}
}
@@ -180,12 +182,12 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
boolean testViaReplicationHandler = random().nextBoolean();
String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();
- try (SolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) {
+ try (HttpSolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) {
// Create a backup.
if (testViaReplicationHandler) {
log.info("Running Backup via replication handler");
BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, coreName, null);
+ CheckBackupStatus checkBackupStatus = new CheckBackupStatus(masterClient, coreName, null);
while (!checkBackupStatus.success) {
checkBackupStatus.fetchStatus();
Thread.sleep(1000);
diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
index d1d3f72..7a7ec78 100644
--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.index.hdfs;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.index.BaseTestCheckIndex;
@@ -66,8 +67,11 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Override
@@ -76,17 +80,21 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
super.setUp();
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
-
directory = new HdfsDirectory(path, conf);
}
@Override
@After
public void tearDown() throws Exception {
- directory.close();
- dfsCluster.getFileSystem().delete(path, true);
- super.tearDown();
+ try {
+ directory.close();
+ } finally {
+ try(FileSystem fs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) {
+ fs.delete(path, true);
+ } finally {
+ super.tearDown();
+ }
+ }
}
@Override
@@ -108,7 +116,7 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
SolrClient client = clients.get(0);
NamedList<Object> response = client.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse();
NamedList<Object> coreInfo = (NamedList<Object>) response.get("core");
- String indexDir = (String) ((NamedList<Object>) coreInfo.get("directory")).get("data") + "/index";
+ String indexDir = ((NamedList<Object>) coreInfo.get("directory")).get("data") + "/index";
args = new String[] {indexDir};
}
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 33caf74..9c97c03 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -82,7 +82,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
} catch (IOException | URISyntaxException e) {
throw new RuntimeException(e);
@@ -257,7 +256,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
- assertTrue(rinfoFuture == null);
+ assertNull(rinfoFuture);
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
@@ -295,7 +294,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
rinfoFuture = ulog.applyBufferedUpdates();
- assertTrue(rinfoFuture != null);
+ assertNotNull(rinfoFuture);
assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
@@ -341,7 +340,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
logReplay.drainPermits();
rinfoFuture = ulog.applyBufferedUpdates();
- assertTrue(rinfoFuture != null);
+ assertNotNull(rinfoFuture);
assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());
// apply a single update
@@ -409,7 +408,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
ulog.bufferUpdates();
assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
- assertTrue(rinfoFuture == null);
+ assertNull(rinfoFuture);
assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
ulog.bufferUpdates();
@@ -737,7 +736,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
//
addDocs(1, start, new LinkedList<Long>()); // don't add this to the versions list because we are going to lose it...
h.close();
- files = HdfsUpdateLog.getLogList(fs, new Path(logDir));;
+ files = HdfsUpdateLog.getLogList(fs, new Path(logDir));
Arrays.sort(files);
FSDataOutputStream dos = fs.create(new Path(new Path(logDir), files[files.length-1]), (short)1);
@@ -850,7 +849,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
Arrays.sort(files);
FSDataOutputStream dos = fs.create(new Path(logDir, files[files.length-1]), (short)1);
- dos.write(new byte[(int)800]); // zero out file
+ dos.write(new byte[800]); // zero out file
dos.close();
@@ -920,34 +919,31 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
Arrays.sort(files);
String fname = files[files.length-1];
- FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]));
- dos.writeLong(0xffffffffffffffffL);
- dos.writeChars("This should be appended to a good log file, representing a bad partially written record.");
- dos.close();
-
- FSDataInputStream dis = fs.open(new Path(logDir, files[files.length-1]));
- byte[] content = new byte[(int)dis.available()];
+ try(FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]))) {
+ dos.writeLong(0xffffffffffffffffL);
+ dos.writeChars("This should be appended to a good log file, representing a bad partially written record.");
+ }
- dis.readFully(content);
+ try(FSDataInputStream dis = fs.open(new Path(logDir, files[files.length-1]))) {
+ byte[] content = new byte[dis.available()];
+ dis.readFully(content);
- dis.close();
+ // Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much!
+ findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content);
+ findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content);
+ findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content);
- // Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much!
- findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content);
- findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content);
- findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content);
+ // WARNING... assumes format of .00000n where n is less than 9
+ long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
+ String fname2 = String.format(Locale.ROOT,
+ UpdateLog.LOG_FILENAME_PATTERN,
+ UpdateLog.TLOG_NAME,
+ logNumber + 1);
- // WARNING... assumes format of .00000n where n is less than 9
- long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
- String fname2 = String.format(Locale.ROOT,
- UpdateLog.LOG_FILENAME_PATTERN,
- UpdateLog.TLOG_NAME,
- logNumber + 1);
-
- dos = fs.create(new Path(logDir, fname2), (short)1);
- dos.write(content);
- dos.close();
-
+ try(FSDataOutputStream dos = fs.create(new Path(logDir, fname2), (short)1)) {
+ dos.write(content);
+ }
+ }
logReplay.release(1000);
logReplayFinish.drainPermits();
@@ -971,9 +967,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
for(;;) {
idx = indexOf(from, data, idx + from.length); // skip over previous match
if (idx < 0) break;
- for (int i=0; i<to.length; i++) {
- data[idx+i] = to[i];
- }
+ System.arraycopy(to, 0, data, idx, to.length);
}
}
diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
index cfdfef5..5ba7f58 100644
--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
@@ -50,14 +50,11 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (URISyntaxException e) {
+ } catch (IOException | URISyntaxException e) {
throw new RuntimeException(e);
}
-
+
System.setProperty("solr.ulog.dir", hdfsUri + "/solr/shard1");
initCore("solrconfig-tlog.xml","schema15.xml");