You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2019/03/08 13:32:00 UTC
[lucene-solr] 01/01: SOLR-13307: Ensure HDFS tests clear System
properties they set
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch jira/solr-13307
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit f78187783a18694af2dade46630612bc90e2d3fb
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Fri Mar 8 08:30:46 2019 -0500
SOLR-13307: Ensure HDFS tests clear System properties they set
Signed-off-by: Kevin Risden <kr...@apache.org>
---
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 15 +-
.../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 4 +-
.../collections/TestHdfsCloudBackupRestore.java | 20 +-
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 14 +-
.../cloud/hdfs/HdfsBasicDistributedZkTest.java | 13 +-
.../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java | 14 +-
.../cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java | 12 +-
.../apache/solr/cloud/hdfs/HdfsNNFailoverTest.java | 11 +-
.../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 14 +-
.../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 12 +-
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 3 +-
.../HdfsTlogReplayBufferedWhileIndexingTest.java | 19 +-
.../cloud/hdfs/HdfsUnloadDistributedZkTest.java | 10 +-
.../hdfs/HdfsWriteToMultipleCollectionsTest.java | 12 +-
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 36 ++-
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 251 ++++++++++-----------
.../solr/handler/TestHdfsBackupRestoreCore.java | 16 +-
.../org/apache/solr/search/TestRecoveryHdfs.java | 52 ++---
.../org/apache/solr/update/TestHdfsUpdateLog.java | 31 ++-
19 files changed, 269 insertions(+), 290 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
index f6e9068..2a26956 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
@@ -63,9 +63,17 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
@AfterClass
public static void teardownClass() throws Exception {
- cluster.shutdown(); // need to close before the MiniDFSCluster
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ shutdownCluster();
+ } finally {
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.blockcache.enabled");
+ }
+ }
}
@Test
@@ -204,5 +212,4 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
solrClient.add(collection, doc);
}
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index b464341..9678c2e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -50,8 +50,8 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
- System.setProperty("tests.hdfs.numdatanodes", "1");
+ System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index c796394..a9901e8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -147,14 +147,17 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
@AfterClass
public static void teardownClass() throws Exception {
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
IOUtils.closeQuietly(fs);
fs = null;
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
}
@Override
@@ -211,11 +214,10 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
assertTrue(expected.contains(d));
}
}
+
@Override
@Test
- // commented 15-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void test() throws Exception {
super.test();
}
-
- }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
index 425f4a9..c06b28a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
@@ -55,9 +55,17 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase {
@AfterClass
public static void teardownClass() throws Exception {
- cluster.shutdown(); // need to close before the MiniDFSCluster
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ shutdownCluster(); // need to close before the MiniDFSCluster
+ } finally {
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.blockcache.enabled");
+ System.clearProperty("solr.hdfs.home");
+ }
+ }
}
public void testDataDirIsNotReused() throws Exception {
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
index 380916a..59293c6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java
@@ -28,13 +28,11 @@ import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
-
@Slow
@Nightly
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// commented 20-July-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest {
private static MiniDFSCluster dfsCluster;
@@ -51,15 +49,16 @@ public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- System.clearProperty("tests.hdfs.numdatanodes");
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("tests.hdfs.numdatanodes");
+ }
}
-
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java
index 076c678..3bdf77a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeyNothingIsSafeTest.java
@@ -33,20 +33,23 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028, https://issues.apache.org/jira/browse/SOLR-10191")
public class HdfsChaosMonkeyNothingIsSafeTest extends ChaosMonkeyNothingIsSafeTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
System.setProperty("solr.hdfs.blockcache.global", "true"); // always use global cache, this test can create a lot of directories
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.blockcache.global");
+ }
}
@Override
@@ -56,12 +59,9 @@ public class HdfsChaosMonkeyNothingIsSafeTest extends ChaosMonkeyNothingIsSafeTe
// super class may hard code directory
useFactory("org.apache.solr.core.HdfsDirectoryFactory");
}
-
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java
index 517d932..ab5eb33 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java
@@ -33,7 +33,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public class HdfsChaosMonkeySafeLeaderTest extends ChaosMonkeySafeLeaderTest {
private static MiniDFSCluster dfsCluster;
@@ -45,8 +44,12 @@ public class HdfsChaosMonkeySafeLeaderTest extends ChaosMonkeySafeLeaderTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.blockcache.global");
+ }
}
@Override
@@ -56,12 +59,9 @@ public class HdfsChaosMonkeySafeLeaderTest extends ChaosMonkeySafeLeaderTest {
// super class may hard code directory
useFactory("org.apache.solr.core.HdfsDirectoryFactory");
}
-
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java
index de92763..b793268 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsNNFailoverTest.java
@@ -35,10 +35,8 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
})
@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class HdfsNNFailoverTest extends BasicDistributedZkTest {
-
private static final String COLLECTION = "collection";
private static MiniDFSCluster dfsCluster;
-
@BeforeClass
public static void setupClass() throws Exception {
@@ -47,8 +45,11 @@ public class HdfsNNFailoverTest extends BasicDistributedZkTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Override
@@ -61,7 +62,7 @@ public class HdfsNNFailoverTest extends BasicDistributedZkTest {
sliceCount = 1;
fixShardCount(TEST_NIGHTLY ? 7 : random().nextInt(2) + 1);
}
-
+
protected String getSolrXml() {
return "solr.xml";
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
index 876b5ee..b881292 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
@@ -51,9 +51,17 @@ public class HdfsRecoveryZkTest extends RecoveryZkTest {
@AfterClass
public static void teardownClass() throws Exception {
- cluster.shutdown(); // need to close before the MiniDFSCluster
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ shutdownCluster(); // need to close before the MiniDFSCluster
+ } finally {
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ System.clearProperty("solr.hdfs.home");
+ }
+ }
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
index 3779dd7..02a7943 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
@@ -33,9 +33,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// 12-Jun-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //2018-03-10
public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest {
-
public HdfsRestartWhileUpdatingTest() throws Exception {
super();
}
@@ -50,14 +48,16 @@ public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ }
}
-
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
index c9771e5..d601565 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
@@ -113,7 +113,8 @@ public class HdfsTestUtil {
System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
- System.setProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean()));
+ System.setProperty("solr.hdfs.blockcache.global",
+ System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean())));
final MiniDFSCluster dfsCluster;
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
index 4986090..52b3a82 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
@@ -30,35 +30,34 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@Slow
@Nightly
-// 12-Jun-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedWhileIndexingTest {
-
+ private static MiniDFSCluster dfsCluster;
+
public HdfsTlogReplayBufferedWhileIndexingTest() throws Exception {
super();
}
- private static MiniDFSCluster dfsCluster;
-
@BeforeClass
public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ }
}
-
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
index 117ec61..c15b8c3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java
@@ -33,7 +33,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-//Commented 4-Oct-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
public class HdfsUnloadDistributedZkTest extends UnloadDistributedZkTest {
private static MiniDFSCluster dfsCluster;
@@ -44,14 +43,15 @@ public class HdfsUnloadDistributedZkTest extends UnloadDistributedZkTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
-
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
index 1905874..6f48b98 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
@@ -57,10 +57,7 @@ import org.junit.Test;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-//Commented 4-Oct-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest {
- private static final String SOLR_HDFS_HOME = "solr.hdfs.home";
- private static final String SOLR_HDFS_BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
private static final String ACOLLECTION = "acollection";
private static MiniDFSCluster dfsCluster;
@@ -72,8 +69,11 @@ public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Override
@@ -168,7 +168,7 @@ public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest {
BlockCache blockCache = ((BlockDirectoryCache) cache)
.getBlockCache();
if (lastBlockCache != null) {
- if (Boolean.getBoolean(SOLR_HDFS_BLOCKCACHE_GLOBAL)) {
+ if (Boolean.getBoolean("solr.hdfs.blockcache.global")) {
assertEquals(lastBlockCache, blockCache);
} else {
assertNotSame(lastBlockCache, blockCache);
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
index b8aa53a..b3d5e27 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
@@ -24,11 +24,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.BasicDistributedZkTest;
@@ -42,14 +40,12 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -58,16 +54,13 @@ import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
@Slow
+@Nightly
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
-@Nightly
public class StressHdfsTest extends BasicDistributedZkTest {
-
private static final String DELETE_DATA_DIR_COLLECTION = "delete_data_dir";
private static MiniDFSCluster dfsCluster;
-
private boolean testRestartIntoSafeMode;
@@ -78,8 +71,11 @@ public class StressHdfsTest extends BasicDistributedZkTest {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Override
@@ -93,13 +89,12 @@ public class StressHdfsTest extends BasicDistributedZkTest {
fixShardCount(TEST_NIGHTLY ? 7 : random().nextInt(2) + 1);
testRestartIntoSafeMode = random().nextBoolean();
}
-
+
protected String getSolrXml() {
return "solr.xml";
}
@Test
- //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
public void test() throws Exception {
randomlyEnableAutoSoftCommit();
@@ -121,7 +116,7 @@ public class StressHdfsTest extends BasicDistributedZkTest {
// enter safe mode and restart a node
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
- int rnd = LuceneTestCase.random().nextInt(10000);
+ int rnd = random().nextInt(10000);
timer.schedule(new TimerTask() {
@@ -140,10 +135,7 @@ public class StressHdfsTest extends BasicDistributedZkTest {
}
}
- private void createAndDeleteCollection() throws SolrServerException,
- IOException, Exception, KeeperException, InterruptedException,
- URISyntaxException {
-
+ private void createAndDeleteCollection() throws Exception {
boolean overshard = random().nextBoolean();
int rep;
int nShards;
@@ -238,11 +230,11 @@ public class StressHdfsTest extends BasicDistributedZkTest {
for (String dataDir : dataDirs) {
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
conf.setBoolean("fs.hdfs.impl.disable.cache", true);
- FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf);
- assertFalse(
- "Data directory exists after collection removal : " + dataDir,
- fs.exists(new Path(dataDir)));
- fs.close();
+ try(FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) {
+ assertFalse(
+ "Data directory exists after collection removal : " + dataDir,
+ fs.exists(new Path(dataDir)));
+ }
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
index 2ea85fd..c599fb7 100644
--- a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
@@ -51,105 +51,96 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
-
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
System.setProperty("solr.hdfs.blockcache.blocksperbank", "1024");
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
}
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- System.clearProperty("solr.hdfs.home");
- System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
+ System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ }
}
@Test
public void testInitArgsOrSysPropConfig() throws Exception {
-
- HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
-
- // test sys prop config
-
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
- hdfsFactory.init(new NamedList<>());
- String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
-
- assertTrue(dataHome.endsWith("/solr1/mock/data"));
-
- System.clearProperty("solr.hdfs.home");
-
- // test init args config
-
- NamedList<Object> nl = new NamedList<>();
- nl.add("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr2");
- hdfsFactory.init(nl);
- dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
-
- assertTrue(dataHome.endsWith("/solr2/mock/data"));
-
- // test sys prop and init args config - init args wins
-
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
- hdfsFactory.init(nl);
- dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
-
- assertTrue(dataHome.endsWith("/solr2/mock/data"));
-
- System.clearProperty("solr.hdfs.home");
-
-
- // set conf dir by sys prop
-
- Path confDir = createTempDir();
-
- System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
-
- Directory dir = hdfsFactory.create(HdfsTestUtil.getURI(dfsCluster) + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT);
- try {
- assertEquals(confDir.toString(), hdfsFactory.getConfDir());
- } finally {
- dir.close();
+ try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
+
+ // test sys prop config
+ System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ hdfsFactory.init(new NamedList<>());
+ String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
+
+ assertTrue(dataHome.endsWith("/solr1/mock/data"));
+
+ System.clearProperty("solr.hdfs.home");
+
+ // test init args config
+ NamedList<Object> nl = new NamedList<>();
+ nl.add("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr2");
+ hdfsFactory.init(nl);
+ dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
+
+ assertTrue(dataHome.endsWith("/solr2/mock/data"));
+
+ // test sys prop and init args config - init args wins
+ System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ hdfsFactory.init(nl);
+ dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
+
+ assertTrue(dataHome.endsWith("/solr2/mock/data"));
+
+ System.clearProperty("solr.hdfs.home");
+
+ // set conf dir by sys prop
+ Path confDir = createTempDir();
+
+ System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
+
+ try (Directory dir = hdfsFactory
+ .create(HdfsTestUtil.getURI(dfsCluster) + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT)) {
+ assertEquals(confDir.toString(), hdfsFactory.getConfDir());
+ }
+
+ // check bool and int getConf impls
+ nl = new NamedList<>();
+ nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
+ System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
+ nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
+ System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
+
+ hdfsFactory.init(nl);
+
+ assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertTrue(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
+
+ nl = new NamedList<>();
+ hdfsFactory.init(nl);
+ System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
+
+ assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertTrue(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
+
+ System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
+ System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
+
+ assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertFalse(hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
}
-
- // check bool and int getConf impls
- nl = new NamedList<>();
- nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
- System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
- nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
- System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
-
- hdfsFactory.init(nl);
-
- assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
- assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
-
- nl = new NamedList<>();
- hdfsFactory.init(nl);
- System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
-
- assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
- assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
-
- System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
- System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
-
- assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
- assertEquals(false, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
-
- hdfsFactory.close();
}
@Test
public void testCleanupOldIndexDirectories() throws Exception {
-
try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
-
System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
@@ -183,56 +174,56 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
conf.set("dfs.permissions.enabled", "false");
Random r = random();
- HdfsDirectoryFactory factory = new HdfsDirectoryFactory();
- SolrMetricManager metricManager = new SolrMetricManager();
- String registry = TestUtil.randomSimpleString(r, 2, 10);
- String scope = TestUtil.randomSimpleString(r,2, 10);
- Map<String,String> props = new HashMap<String,String>();
- props.put(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr");
- props.put(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
- props.put(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_ENABLE, "false");
- props.put(HdfsDirectoryFactory.LOCALITYMETRICS_ENABLED, "true");
- factory.init(new NamedList<>(props));
- factory.initializeMetrics(metricManager, registry, "foo", scope);
-
- // get the metrics map for the locality bean
- MetricsMap metrics = (MetricsMap)((SolrMetricManager.GaugeWrapper)metricManager.registry(registry).getMetrics().get("OTHER." + scope + ".hdfsLocality")).getGauge();
- // We haven't done anything, so there should be no data
- Map<String,Object> statistics = metrics.getValue();
- assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0l,
- statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
- assertEquals(
- "Counted bytes as local when none written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO), 0,
- statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO));
-
- // create a directory and a file
- String path = HdfsTestUtil.getURI(dfsCluster) + "/solr3/";
- Directory dir = factory.create(path, NoLockFactory.INSTANCE, DirContext.DEFAULT);
- try(IndexOutput writer = dir.createOutput("output", null)) {
- writer.writeLong(42l);
+ try(HdfsDirectoryFactory factory = new HdfsDirectoryFactory()) {
+ SolrMetricManager metricManager = new SolrMetricManager();
+ String registry = TestUtil.randomSimpleString(r, 2, 10);
+ String scope = TestUtil.randomSimpleString(r, 2, 10);
+ Map<String, String> props = new HashMap<String, String>();
+ props.put(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr");
+ props.put(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
+ props.put(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_ENABLE, "false");
+ props.put(HdfsDirectoryFactory.LOCALITYMETRICS_ENABLED, "true");
+ factory.init(new NamedList<>(props));
+ factory.initializeMetrics(metricManager, registry, "foo", scope);
+
+ // get the metrics map for the locality bean
+ MetricsMap metrics = (MetricsMap) ((SolrMetricManager.GaugeWrapper) metricManager.registry(registry).getMetrics().get("OTHER." + scope + ".hdfsLocality")).getGauge();
+ // We haven't done anything, so there should be no data
+ Map<String, Object> statistics = metrics.getValue();
+ assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0l,
+ statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
+ assertEquals(
+ "Counted bytes as local when none written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO), 0,
+ statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO));
+
+ // create a directory and a file
+ String path = HdfsTestUtil.getURI(dfsCluster) + "/solr3/";
+ try (Directory dir = factory.create(path, NoLockFactory.INSTANCE, DirContext.DEFAULT)) {
+ try (IndexOutput writer = dir.createOutput("output", null)) {
+ writer.writeLong(42L);
+ }
+
+ final long long_bytes = Long.SIZE / Byte.SIZE;
+
+ // no locality because hostname not set
+ factory.setHost("bogus");
+ statistics = metrics.getValue();
+ assertEquals("Wrong number of total bytes counted: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
+ long_bytes, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
+ assertEquals("Wrong number of total blocks counted: " + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL),
+ 1, statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL));
+ assertEquals(
+ "Counted block as local when bad hostname set: " + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL),
+ 0, statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL));
+
+ // set hostname and check again
+ factory.setHost("127.0.0.1");
+ statistics = metrics.getValue();
+ assertEquals(
+ "Did not count block as local after setting hostname: "
+ + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL),
+ long_bytes, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL));
+ }
}
-
- final long long_bytes = Long.SIZE / Byte.SIZE;
-
- // no locality because hostname not set
- factory.setHost("bogus");
- statistics = metrics.getValue();
- assertEquals("Wrong number of total bytes counted: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL),
- long_bytes, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL));
- assertEquals("Wrong number of total blocks counted: " + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL),
- 1, statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_TOTAL));
- assertEquals(
- "Counted block as local when bad hostname set: " + statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL),
- 0, statistics.get(HdfsLocalityReporter.LOCALITY_BLOCKS_LOCAL));
-
- // set hostname and check again
- factory.setHost("127.0.0.1");
- statistics = metrics.getValue();
- assertEquals(
- "Did not count block as local after setting hostname: "
- + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL),
- long_bytes, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_LOCAL));
-
- factory.close();
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
index 038e451..60f77b4 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.solr.handler;
import java.io.IOException;
@@ -145,14 +144,17 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
@AfterClass
public static void teardownClass() throws Exception {
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
IOUtils.closeQuietly(fs);
fs = null;
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
}
@Test
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 75f3012..33caf74 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.search;
-
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
import java.io.IOException;
@@ -65,18 +64,14 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
// TODO: longer term this should be combined with TestRecovery somehow ??
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class TestRecoveryHdfs extends SolrTestCaseJ4 {
-
// means that we've seen the leader and have version info (i.e. we are a non-leader replica)
private static String FROM_LEADER = DistribPhase.FROMLEADER.toString();
private static int timeout=60; // acquire timeout in seconds. change this to a huge number when debugging to prevent threads from advancing.
private static MiniDFSCluster dfsCluster;
-
private static String hdfsUri;
-
private static FileSystem fs;
@BeforeClass
@@ -100,16 +95,21 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
@AfterClass
public static void afterClass() throws Exception {
- System.clearProperty("solr.ulog.dir");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- deleteCore();
IOUtils.closeQuietly(fs);
fs = null;
- HdfsTestUtil.teardownClass(dfsCluster);
-
- hdfsDataDir = null;
- dfsCluster = null;
+ try {
+ deleteCore();
+ } finally {
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ hdfsDataDir = null;
+ System.clearProperty("solr.ulog.dir");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
+ }
}
@Test
@@ -135,11 +135,9 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
assertTrue("Expected to find tlogs with a replication factor of 2", foundRep2);
}
-
@Test
public void testLogReplay() throws Exception {
try {
-
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
@@ -154,7 +152,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
clearIndex();
assertU(commit());
@@ -230,12 +227,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
UpdateLog.testing_logReplayHook = null;
UpdateLog.testing_logReplayFinishHook = null;
}
-
}
@Test
public void testBuffering() throws Exception {
-
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
@@ -250,7 +245,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;
-
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
@@ -383,14 +377,11 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
req().close();
}
-
}
-
@Test
@Ignore("HDFS-3107: no truncate support yet")
public void testDropBuffered() throws Exception {
-
DirectUpdateHandler2.commitOnClose = false;
final Semaphore logReplay = new Semaphore(0);
final Semaphore logReplayFinish = new Semaphore(0);
@@ -499,12 +490,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
+"]"
);
-
updateJ(jsonAdd(sdoc("id","C2", "_version_","302")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
-
-
-
assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
DirectUpdateHandler2.commitOnClose = true;
@@ -513,13 +500,10 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
req().close();
}
-
}
-
@Test
public void testExistOldBufferLog() throws Exception {
-
DirectUpdateHandler2.commitOnClose = false;
SolrQueryRequest req = req();
@@ -587,11 +571,8 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
req().close();
}
-
}
-
-
// make sure that on a restart, versions don't start too low
@Test
public void testVersionsOnRestart() throws Exception {
@@ -616,7 +597,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
assertJQ(req("qt","/get", "getVersions","2")
,"/versions==[" + v2 + "," + v1a + "]"
);
-
}
// make sure that log isn't needlessly replayed after a clean close
@@ -636,7 +616,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
-
SolrQueryRequest req = req();
UpdateHandler uhandler = req.getCore().getUpdateHandler();
UpdateLog ulog = uhandler.getUpdateLog();
@@ -669,7 +648,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
}
-
private void addDocs(int nDocs, int start, LinkedList<Long> versions) throws Exception {
for (int i=0; i<nDocs; i++) {
versions.addFirst( addAndGetVersion( sdoc("id",Integer.toString(start + nDocs)) , null) );
@@ -783,7 +761,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// test that a partially written last tlog entry (that will cause problems for both reverse reading and for
// log replay) doesn't stop us from coming up, and from recovering the documents that were not cut off.
//
-
@Test
public void testTruncatedLog() throws Exception {
try {
@@ -850,7 +827,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
}
-
//
// test that a corrupt tlog doesn't stop us from coming up
//
@@ -912,8 +888,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
}
}
-
-
// in rare circumstances, two logs can be left uncapped (lacking a commit at the end signifying that all the content in the log was committed)
@Test
public void testRecoveryMultipleLogs() throws Exception {
diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
index 62bd1d8..cfdfef5 100644
--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
@@ -37,13 +37,9 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
-
private static MiniDFSCluster dfsCluster;
-
private static String hdfsUri;
-
private static FileSystem fs;
@BeforeClass
@@ -69,22 +65,25 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
@AfterClass
public static void afterClass() throws Exception {
- System.clearProperty("solr.ulog.dir");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
- deleteCore();
IOUtils.closeQuietly(fs);
fs = null;
- HdfsTestUtil.teardownClass(dfsCluster);
-
- hdfsDataDir = null;
- dfsCluster = null;
+ try {
+ deleteCore();
+ } finally {
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ hdfsDataDir = null;
+ dfsCluster = null;
+ System.clearProperty("solr.ulog.dir");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
+ }
}
@Test
- //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
public void testFSThreadSafety() throws Exception {
-
final SolrQueryRequest req = req();
final UpdateHandler uhandler = req.getCore().getUpdateHandler();
((DirectUpdateHandler2) uhandler).getCommitTracker().setTimeUpperBound(100);
@@ -131,15 +130,11 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
}
}
};
-
-
thread.start();
thread2.start();
thread.join();
thread2.join();
-
}
-
}