You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2015/01/13 16:12:47 UTC
svn commit: r1651373 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/core/ core/src/test/org/apache/solr/cloud/hdfs/
core/src/test/org/apache/solr/core/ core/src/test/org/apache/solr/util/
server/solr/configsets/sample_techproducts_config...
Author: markrmiller
Date: Tue Jan 13 15:12:46 2015
New Revision: 1651373
URL: http://svn.apache.org/r1651373
Log:
SOLR-6943: HdfsDirectoryFactory should fall back to system props for most of it's config if it is not found in solrconfig.xml.
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java
lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1651373&r1=1651372&r2=1651373&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Jan 13 15:12:46 2015
@@ -691,6 +691,9 @@ Other Changes
* SOLR-6904: Removed deprecated Circle & rect syntax. See upgrading notes. (David Smiley)
+* SOLR-6943: HdfsDirectoryFactory should fall back to system props for most of it's config
+ if it is not found in solrconfig.xml. (Mark Miller, Mike Drob)
+
================== 4.10.3 ==================
Bug Fixes
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java?rev=1651373&r1=1651372&r2=1651373&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Tue Jan 13 15:12:46 2015
@@ -61,7 +61,7 @@ public class HdfsDirectoryFactory extend
public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
- public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled";
+ public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB = "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
@@ -96,13 +96,13 @@ public class HdfsDirectoryFactory extend
@Override
public void init(NamedList args) {
params = SolrParams.toSolrParams(args);
- this.hdfsDataDir = params.get(HDFS_HOME);
+ this.hdfsDataDir = getConfig(HDFS_HOME, null);
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
this.hdfsDataDir = null;
} else {
LOG.info(HDFS_HOME + "=" + this.hdfsDataDir);
}
- boolean kerberosEnabled = params.getBool(KERBEROS_ENABLED, false);
+ boolean kerberosEnabled = getConfig(KERBEROS_ENABLED, false);
LOG.info("Solr Kerberos Authentication "
+ (kerberosEnabled ? "enabled" : "disabled"));
if (kerberosEnabled) {
@@ -132,6 +132,7 @@ public class HdfsDirectoryFactory extend
@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
+ assert params != null : "init must be called before create";
LOG.info("creating directory factory for path {}", path);
Configuration conf = getConf();
@@ -139,25 +140,19 @@ public class HdfsDirectoryFactory extend
metrics = MetricsHolder.metrics;
}
- boolean blockCacheEnabled = params.getBool(BLOCKCACHE_ENABLED, true);
- boolean blockCacheGlobal = params.getBool(BLOCKCACHE_GLOBAL, false); // default to false for back compat
- boolean blockCacheReadEnabled = params.getBool(BLOCKCACHE_READ_ENABLED, true);
- boolean blockCacheWriteEnabled = params.getBool(BLOCKCACHE_WRITE_ENABLED, false);
-
- if (blockCacheWriteEnabled) {
- LOG.warn("Using " + BLOCKCACHE_WRITE_ENABLED + " is currently buggy and can result in readers seeing a corrupted view of the index.");
- }
+ boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
+ boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, false); // default to false for back compat
+ boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
final Directory dir;
if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
- int numberOfBlocksPerBank = params.getInt(NUMBEROFBLOCKSPERBANK, 16384);
+ int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
int blockSize = BlockDirectory.BLOCK_SIZE;
- int bankCount = params.getInt(BLOCKCACHE_SLAB_COUNT, 1);
+ int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
- boolean directAllocation = params.getBool(
- BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
+ boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
int slabSize = numberOfBlocksPerBank * blockSize;
LOG.info(
@@ -168,8 +163,8 @@ public class HdfsDirectoryFactory extend
new Object[] {slabSize, bankCount,
((long) bankCount * (long) slabSize)});
- int bufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", 128);
- int bufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
+ int bufferSize = getConfig("solr.hdfs.blockcache.bufferstore.buffersize", 128);
+ int bufferCount = getConfig("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
BlockCache blockCache = getBlockDirectoryCache(numberOfBlocksPerBank,
blockSize, bankCount, directAllocation, slabSize,
@@ -177,25 +172,51 @@ public class HdfsDirectoryFactory extend
Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
HdfsDirectory hdfsDirectory = new HdfsDirectory(new Path(path), lockFactory, conf);
- dir = new BlockDirectory(path, hdfsDirectory, cache, null,
- blockCacheReadEnabled, blockCacheWriteEnabled);
+ dir = new BlockDirectory(path, hdfsDirectory, cache, null, blockCacheReadEnabled, false);
} else {
dir = new HdfsDirectory(new Path(path), lockFactory, conf);
}
- boolean nrtCachingDirectory = params.getBool(NRTCACHINGDIRECTORY_ENABLE, true);
+ boolean nrtCachingDirectory = getConfig(NRTCACHINGDIRECTORY_ENABLE, true);
if (nrtCachingDirectory) {
- double nrtCacheMaxMergeSizeMB = params.getInt(
- NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
- double nrtCacheMaxCacheMB = params.getInt(NRTCACHINGDIRECTORY_MAXCACHEMB,
- 192);
+ double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
+ double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
- return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB,
- nrtCacheMaxCacheMB);
+ return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
}
return dir;
}
+ boolean getConfig(String name, boolean defaultValue) {
+ Boolean value = params.getBool(name);
+ if (value == null) {
+ String sysValue = System.getProperty(name);
+ if (sysValue != null) {
+ value = Boolean.valueOf(sysValue);
+ }
+ }
+ return value == null ? defaultValue : value;
+ }
+
+ int getConfig(String name, int defaultValue) {
+ Integer value = params.getInt(name);
+ if (value == null) {
+ String sysValue = System.getProperty(name);
+ if (sysValue != null) {
+ value = Integer.parseInt(sysValue);
+ }
+ }
+ return value == null ? defaultValue : value;
+ }
+
+ String getConfig(String name, String defaultValue) {
+ String value = params.get(name);
+ if (value == null) {
+ value = System.getProperty(name);
+ }
+ return value == null ? defaultValue : value;
+ }
+
private BlockCache getBlockDirectoryCache(int numberOfBlocksPerBank, int blockSize, int bankCount,
boolean directAllocation, int slabSize, int bufferSize, int bufferCount, boolean staticBlockCache) {
if (!staticBlockCache) {
@@ -252,7 +273,7 @@ public class HdfsDirectoryFactory extend
private Configuration getConf() {
Configuration conf = new Configuration();
- confDir = params.get(CONFIG_DIRECTORY, null);
+ confDir = getConfig(CONFIG_DIRECTORY, null);
HdfsUtil.addHdfsResources(conf, confDir);
return conf;
}
@@ -329,12 +350,12 @@ public class HdfsDirectoryFactory extend
}
private void initKerberos() {
- String keytabFile = params.get(KERBEROS_KEYTAB, "").trim();
+ String keytabFile = getConfig(KERBEROS_KEYTAB, "").trim();
if (keytabFile.length() == 0) {
throw new IllegalArgumentException(KERBEROS_KEYTAB + " required because "
+ KERBEROS_ENABLED + " set to true");
}
- String principal = params.get(KERBEROS_PRINCIPAL, "");
+ String principal = getConfig(KERBEROS_PRINCIPAL, "");
if (principal.length() == 0) {
throw new IllegalArgumentException(KERBEROS_PRINCIPAL
+ " required because " + KERBEROS_ENABLED + " set to true");
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java?rev=1651373&r1=1651372&r2=1651373&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Tue Jan 13 15:12:46 2015
@@ -39,6 +39,10 @@ public class HdfsTestUtil {
private static Map<MiniDFSCluster,Timer> timers = new ConcurrentHashMap<>();
public static MiniDFSCluster setupClass(String dir) throws Exception {
+ return setupClass(dir, true);
+ }
+
+ public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting) throws Exception {
LuceneTestCase.assumeFalse("HDFS tests were disabled by -Dtests.disableHdfs",
Boolean.parseBoolean(System.getProperty("tests.disableHdfs", "false")));
@@ -68,20 +72,21 @@ public class HdfsTestUtil {
System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
-
- NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
-
- int rnd = LuceneTestCase.random().nextInt(10000);
- Timer timer = new Timer();
- timer.schedule(new TimerTask() {
+ if (safeModeTesting) {
+ NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
- @Override
- public void run() {
- NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
- }
- }, rnd);
-
- timers.put(dfsCluster, timer);
+ int rnd = LuceneTestCase.random().nextInt(10000);
+ Timer timer = new Timer();
+ timer.schedule(new TimerTask() {
+
+ @Override
+ public void run() {
+ NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
+ }
+ }, rnd);
+
+ timers.put(dfsCluster, timer);
+ }
SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory");
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java?rev=1651373&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java Tue Jan 13 15:12:46 2015
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.nio.file.Path;
+
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.NoLockFactory;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.util.MockCoreContainer.MockCoreDescriptor;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
+
+@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
+public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
+
+ private static MiniDFSCluster dfsCluster;
+
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
+ }
+
+ @AfterClass
+ public static void teardownClass() throws Exception {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
+ dfsCluster = null;
+ }
+
+ @Test
+ public void testInitArgsOrSysPropConfig() throws Exception {
+
+ HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
+
+ // test sys prop config
+
+ System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr1");
+ hdfsFactory.init(new NamedList<>());
+ String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
+
+ assertTrue(dataHome.endsWith("/solr1/mock/data"));
+
+ System.clearProperty("solr.hdfs.home");
+
+ // test init args config
+
+ NamedList<Object> nl = new NamedList<>();
+ nl.add("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr2");
+ hdfsFactory.init(nl);
+ dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
+
+ assertTrue(dataHome.endsWith("/solr2/mock/data"));
+
+ // test sys prop and init args config - init args wins
+
+ System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr1");
+ hdfsFactory.init(nl);
+ dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
+
+ assertTrue(dataHome.endsWith("/solr2/mock/data"));
+
+ System.clearProperty("solr.hdfs.home");
+
+
+ // set conf dir by sys prop
+
+ Path confDir = createTempDir();
+
+ System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
+
+ Directory dir = hdfsFactory.create(dfsCluster.getURI().toString() + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT);
+ try {
+ assertEquals(confDir.toString(), hdfsFactory.getConfDir());
+ } finally {
+ dir.close();
+ }
+
+ // check bool and int getConf impls
+ nl = new NamedList<>();
+ nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
+ System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
+ nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
+ System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
+
+ hdfsFactory.init(nl);
+
+ assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
+
+ nl = new NamedList<>();
+ hdfsFactory.init(nl);
+ System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
+
+ assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
+
+ System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
+ System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
+
+ assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
+ assertEquals(false, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
+
+ hdfsFactory.close();
+ }
+
+}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java?rev=1651373&r1=1651372&r2=1651373&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java Tue Jan 13 15:12:46 2015
@@ -1,7 +1,5 @@
package org.apache.solr.util;
-import org.apache.solr.core.CoreContainer;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -19,6 +17,22 @@ import org.apache.solr.core.CoreContaine
* limitations under the License.
*/
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
+
+
public class MockCoreContainer extends CoreContainer {
+ public static class MockCoreDescriptor extends CoreDescriptor {
+ public MockCoreDescriptor() {
+ super(new MockCoreContainer(), "mock", "path");
+ }
+ }
+
+ public MockCoreContainer() {
+ super(new Object());
+ }
+ public String getCoreRootDirectory() {
+ return "coreroot";
+ }
}
Modified: lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml?rev=1651373&r1=1651372&r2=1651373&view=diff
==============================================================================
--- lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml Tue Jan 13 15:12:46 2015
@@ -117,23 +117,7 @@
persistent, and doesn't work with replication.
-->
<directoryFactory name="DirectoryFactory"
- class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}">
-
-
- <!-- These will be used if you are using the solr.HdfsDirectoryFactory,
- otherwise they will be ignored. If you don't plan on using hdfs,
- you can safely remove this section. -->
- <!-- The root directory that collection data should be written to. -->
- <str name="solr.hdfs.home">${solr.hdfs.home:}</str>
- <!-- The hadoop configuration files to use for the hdfs client. -->
- <str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
- <!-- Enable/Disable the hdfs cache. -->
- <str name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</str>
- <!-- Enable/Disable using one global cache for all SolrCores.
- The settings used will be from the first HdfsDirectoryFactory created. -->
- <str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
-
- </directoryFactory>
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
<!-- The CodecFactory for defining the format of the inverted index.
The default implementation is SchemaCodecFactory, which is the official Lucene