You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2019/03/08 18:34:42 UTC
[lucene-solr] branch jira/solr-13307 updated: Make blocksperbank
and fs.hdfs.impl.disable.cache consistent
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch jira/solr-13307
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/solr-13307 by this push:
new db455ae Make blocksperbank and fs.hdfs.impl.disable.cache consistent
db455ae is described below
commit db455ae694089fb58d5ba04ec6aeb31a5cc52c41
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Fri Mar 8 13:34:27 2019 -0500
Make blocksperbank and fs.hdfs.impl.disable.cache consistent
---
.../solr/cloud/MoveReplicaHDFSFailoverTest.java | 3 --
.../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 2 --
.../HdfsCollectionsAPIDistributedZkTest.java | 2 --
.../collections/TestHdfsCloudBackupRestore.java | 3 --
.../HdfsAutoAddReplicasIntegrationTest.java | 2 --
.../solr/cloud/hdfs/HdfsRecoverLeaseTest.java | 6 ++--
.../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 2 --
.../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 2 --
.../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 24 ++++++++-----
.../apache/solr/cloud/hdfs/HdfsThreadLeakTest.java | 41 ++++++++--------------
.../HdfsTlogReplayBufferedWhileIndexingTest.java | 2 --
.../hdfs/HdfsWriteToMultipleCollectionsTest.java | 8 ++---
.../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 1 -
.../apache/solr/core/HdfsDirectoryFactoryTest.java | 20 +++++------
.../solr/handler/TestHdfsBackupRestoreCore.java | 3 --
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 22 +++++++-----
.../org/apache/solr/search/TestRecoveryHdfs.java | 3 --
.../org/apache/solr/update/TestHdfsUpdateLog.java | 3 --
18 files changed, 59 insertions(+), 90 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
index 2a26956..f64bb40 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.solr.cloud;
import java.io.IOException;
@@ -52,7 +51,6 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
.configure();
- System.setProperty("solr.hdfs.blockcache.enabled", "false");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
ZkConfigManager configManager = new ZkConfigManager(zkClient());
@@ -71,7 +69,6 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase {
} finally {
dfsCluster = null;
System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.blockcache.enabled");
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index 9678c2e..f1e6e05 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -39,7 +39,6 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -50,7 +49,6 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
index 700165b..d3e8da6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java
@@ -38,7 +38,6 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -49,7 +48,6 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index a9901e8..bd99075 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -112,7 +112,6 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
if (fs instanceof DistributedFileSystem) {
@@ -155,8 +154,6 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
dfsCluster = null;
System.clearProperty("solr.hdfs.home");
System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
index 37ce805..fb13e21 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
@@ -39,7 +39,6 @@ public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrati
@BeforeClass
public static void setupClass() throws Exception {
System.setProperty("solr.hdfs.blockcache.global", "true");
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "512");
System.setProperty("tests.hdfs.numdatanodes", "1");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -51,7 +50,6 @@ public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrati
} finally {
dfsCluster = null;
System.clearProperty("solr.hdfs.blockcache.global");
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
System.clearProperty("tests.hdfs.numdatanodes");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
index fcacb8b..ad8556c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java
@@ -72,8 +72,7 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
URI uri = dfsCluster.getURI();
Path path = new Path(uri);
- Configuration conf = new Configuration();
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
FileSystem fs1 = FileSystem.get(path.toUri(), conf);
Path testFile = new Path(uri.toString() + "/testfile");
FSDataOutputStream out = fs1.create(testFile);
@@ -131,8 +130,7 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
final URI uri = dfsCluster.getURI();
final Path path = new Path(uri);
- final Configuration conf = new Configuration();
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ final Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
// n threads create files
class WriterThread extends Thread {
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
index b881292..dba1b56 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java
@@ -41,7 +41,6 @@ public class HdfsRecoveryZkTest extends RecoveryZkTest {
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
ZkConfigManager configManager = new ZkConfigManager(zkClient());
configManager.uploadConfigDir(configset("cloud-hdfs"), "conf");
@@ -58,7 +57,6 @@ public class HdfsRecoveryZkTest extends RecoveryZkTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
System.clearProperty("solr.hdfs.home");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
index 02a7943..3584e3d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java
@@ -43,7 +43,6 @@ public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest {
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
}
@AfterClass
@@ -52,7 +51,6 @@ public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
index d601565..7b94d3d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
@@ -43,6 +43,7 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.HdfsDirectoryFactory;
import org.apache.solr.util.HdfsUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,15 +114,16 @@ public class HdfsTestUtil {
System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
- System.setProperty("solr.hdfs.blockcache.global",
- System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean())));
+ System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_GLOBAL,
+ System.getProperty(HdfsDirectoryFactory.BLOCKCACHE_GLOBAL, Boolean.toString(LuceneTestCase.random().nextBoolean())));
+ System.setProperty(HdfsDirectoryFactory.NUMBEROFBLOCKSPERBANK, "2048");
final MiniDFSCluster dfsCluster;
if (!haTesting) {
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes).format(true).build();
- System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, getDataDir(dfsCluster, "solr_hdfs_home"));
} else {
dfsCluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(dataNodes)
@@ -130,7 +132,7 @@ public class HdfsTestUtil {
Configuration haConfig = getClientConfiguration(dfsCluster);
HdfsUtil.TEST_CONF = haConfig;
- System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, getDataDir(dfsCluster, "solr_hdfs_home"));
}
dfsCluster.waitActive();
@@ -186,13 +188,16 @@ public class HdfsTestUtil {
}
public static Configuration getClientConfiguration(MiniDFSCluster dfsCluster) {
+ Configuration conf;
if (dfsCluster.getNameNodeInfos().length > 1) {
- Configuration conf = new Configuration();
+ conf = new Configuration();
HATestUtil.setFailoverConfigurations(dfsCluster, conf);
- return conf;
+
} else {
- return new Configuration();
+ conf = new Configuration();
}
+ conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+ return conf;
}
public static void teardownClass(MiniDFSCluster dfsCluster) throws Exception {
@@ -227,8 +232,9 @@ public class HdfsTestUtil {
System.clearProperty("solr.lock.type");
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
- System.clearProperty("solr.hdfs.home");
- System.clearProperty("solr.hdfs.blockcache.global");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
+ System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_GLOBAL);
+ System.clearProperty(HdfsDirectoryFactory.NUMBEROFBLOCKSPERBANK);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
index 1cd6819..82e1501 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java
@@ -37,7 +37,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
-
private static MiniDFSCluster dfsCluster;
@BeforeClass
@@ -47,38 +46,26 @@ public class HdfsThreadLeakTest extends SolrTestCaseJ4 {
@AfterClass
public static void afterClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
- }
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
- }
-
- @After
- public void tearDown() throws Exception {
- super.tearDown();
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Test
public void testBasic() throws IOException {
String uri = HdfsTestUtil.getURI(dfsCluster);
Path path = new Path(uri);
- Configuration conf = new Configuration();
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
- FileSystem fs = FileSystem.get(path.toUri(), conf);
- Path testFile = new Path(uri.toString() + "/testfile");
- FSDataOutputStream out = fs.create(testFile);
-
- out.write(5);
- out.hflush();
- out.close();
-
- ((DistributedFileSystem) fs).recoverLease(testFile);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ try(FileSystem fs = FileSystem.get(path.toUri(), conf)) {
+ Path testFile = new Path(uri + "/testfile");
+ try (FSDataOutputStream out = fs.create(testFile)) {
+ out.write(5);
+ out.hflush();
+ }
-
- fs.close();
+ ((DistributedFileSystem) fs).recoverLease(testFile);
+ }
}
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
index 52b3a82..aab970c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java
@@ -42,7 +42,6 @@ public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedW
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
}
@@ -52,7 +51,6 @@ public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedW
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
index 6f48b98..cc6381f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java
@@ -141,10 +141,10 @@ public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest {
Directory dir = factory.get(core.getDataDir(), null, null);
try {
long dataDirSize = factory.size(dir);
- FileSystem fileSystem = null;
-
- fileSystem = FileSystem.newInstance(
- new Path(core.getDataDir()).toUri(), new Configuration());
+
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ FileSystem fileSystem = FileSystem.newInstance(
+ new Path(core.getDataDir()).toUri(), conf);
long size = fileSystem.getContentSummary(
new Path(core.getDataDir())).getLength();
assertEquals(size, dataDirSize);
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
index b3d5e27..04f9387 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
@@ -229,7 +229,6 @@ public class StressHdfsTest extends BasicDistributedZkTest {
// check that all dirs are gone
for (String dataDir : dataDirs) {
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
try(FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) {
assertFalse(
"Data directory exists after collection removal : " + dataDir,
diff --git a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
index c599fb7..d99eec3 100644
--- a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
@@ -55,7 +55,6 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@BeforeClass
public static void setupClass() throws Exception {
- System.setProperty("solr.hdfs.blockcache.blocksperbank", "1024");
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
}
@@ -65,9 +64,10 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
HdfsTestUtil.teardownClass(dfsCluster);
} finally {
dfsCluster = null;
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
+ System.clearProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY);
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
- System.clearProperty("solr.hdfs.blockcache.blocksperbank");
+ System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
}
}
@@ -76,13 +76,13 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
// test sys prop config
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr1/mock/data"));
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
// test init args config
NamedList<Object> nl = new NamedList<>();
@@ -93,13 +93,13 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
assertTrue(dataHome.endsWith("/solr2/mock/data"));
// test sys prop and init args config - init args wins
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(nl);
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr2/mock/data"));
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
// set conf dir by sys prop
Path confDir = createTempDir();
@@ -141,11 +141,11 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
@Test
public void testCleanupOldIndexDirectories() throws Exception {
try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) {
- System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1");
+ System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr1/mock/data"));
- System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.HDFS_HOME);
FileSystem hdfs = dfsCluster.getFileSystem();
@@ -178,7 +178,7 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
SolrMetricManager metricManager = new SolrMetricManager();
String registry = TestUtil.randomSimpleString(r, 2, 10);
String scope = TestUtil.randomSimpleString(r, 2, 10);
- Map<String, String> props = new HashMap<String, String>();
+ Map<String, String> props = new HashMap<>();
props.put(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr");
props.put(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
props.put(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_ENABLE, "false");
diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
index 60f77b4..72bcdb0 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
@@ -109,7 +109,6 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
if (fs instanceof DistributedFileSystem) {
@@ -152,8 +151,6 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
dfsCluster = null;
System.clearProperty("solr.hdfs.home");
System.clearProperty("solr.hdfs.default.backup.path");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
}
}
diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
index d1d3f72..de45695 100644
--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
@@ -42,7 +42,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
private static MiniDFSCluster dfsCluster;
private static Path path;
@@ -66,8 +65,11 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
@AfterClass
public static void teardownClass() throws Exception {
- HdfsTestUtil.teardownClass(dfsCluster);
- dfsCluster = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ }
}
@Override
@@ -76,17 +78,21 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
super.setUp();
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
-
directory = new HdfsDirectory(path, conf);
}
@Override
@After
public void tearDown() throws Exception {
- directory.close();
- dfsCluster.getFileSystem().delete(path, true);
- super.tearDown();
+ try {
+ directory.close();
+ } finally {
+ try {
+ dfsCluster.getFileSystem().delete(path, true);
+ } finally {
+ super.tearDown();
+ }
+ }
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 33caf74..035a0b02 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -82,7 +82,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
} catch (IOException | URISyntaxException e) {
throw new RuntimeException(e);
@@ -106,8 +105,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
dfsCluster = null;
hdfsDataDir = null;
System.clearProperty("solr.ulog.dir");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
index cfdfef5..37cb6be 100644
--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
@@ -50,7 +50,6 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
try {
URI uri = new URI(hdfsUri);
Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
- conf.setBoolean("fs.hdfs.impl.disable.cache", true);
fs = FileSystem.get(uri, conf);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -76,8 +75,6 @@ public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
hdfsDataDir = null;
dfsCluster = null;
System.clearProperty("solr.ulog.dir");
- System.clearProperty("test.build.data");
- System.clearProperty("test.cache.data");
}
}
}