You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2012/07/07 23:48:20 UTC
svn commit: r1358650 - in /lucene/dev/trunk:
lucene/core/src/java/org/apache/lucene/store/ solr/
solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/search/
solr/core/src/test/org/apache/solr/cloud/ solr/core/src/test/org/apache...
Author: yonik
Date: Sat Jul 7 21:48:19 2012
New Revision: 1358650
URL: http://svn.apache.org/viewvc?rev=1358650&view=rev
Log:
SOLR-2702: add replication support to NRTCachingDirectoryFactory and make it the default
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestSolrJ.java (with props)
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java
lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml
lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java Sat Jul 7 21:48:19 2012
@@ -84,6 +84,10 @@ public class NRTCachingDirectory extends
maxCachedBytes = (long) (maxCachedMB*1024*1024);
}
+ public Directory getDelegate() {
+ return delegate;
+ }
+
@Override
public LockFactory getLockFactory() {
return delegate.getLockFactory();
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sat Jul 7 21:48:19 2012
@@ -47,6 +47,11 @@ New Features
* SOLR-3562: Add options to remove instance dir or data dir on core unload.
(Mark Miller, Per Steffensen)
+* SOLR-2702: The default directory factory was changed to NRTCachingDirectoryFactory
+ which wraps the StandardDirectoryFactory and caches small files for improved
+ Near Real-time (NRT) performance. (Mark Miller, yonik)
+
+
Bug Fixes
* SOLR-3582: Our ZooKeeper watchers respond to session events as if they are change events,
@@ -75,6 +80,7 @@ Other Changes
* SOLR-3599: Add zkClientTimeout to solr.xml so that it's obvious how to change it and so
that you can change it with a system property. (Mark Miller)
+
================== 4.0.0-ALPHA ==================
More information about this release, including any errata related to the
release notes, upgrade instructions, or other changes may be found online at:
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java Sat Jul 7 21:48:19 2012
@@ -385,7 +385,7 @@ public final class SolrCore implements S
dirFactory = getResourceLoader().newInstance(info.className, DirectoryFactory.class);
dirFactory.init(info.initArgs);
} else {
- dirFactory = new StandardDirectoryFactory();
+ dirFactory = new NRTCachingDirectoryFactory();
}
// And set it
directoryFactory = dirFactory;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Sat Jul 7 21:48:19 2012
@@ -37,6 +37,7 @@ import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
@@ -118,6 +119,18 @@ public class SolrIndexSearcher extends I
this(core, schema,name, core.getIndexReaderFactory().newReader(directoryFactory.get(path, config.lockType), core), true, enableCache, false, directoryFactory);
}
+ private static String getIndexDir(Directory dir) {
+ if (dir instanceof FSDirectory) {
+ return ((FSDirectory)dir).getDirectory().getAbsolutePath();
+ } else if (dir instanceof NRTCachingDirectory) {
+ // recurse on the delegate
+ return getIndexDir(((NRTCachingDirectory) dir).getDelegate());
+ } else {
+ log.warn("WARNING: Directory impl does not support setting indexDir: " + dir.getClass().getName());
+ return null;
+ }
+ }
+
public SolrIndexSearcher(SolrCore core, IndexSchema schema, String name, DirectoryReader r, boolean closeReader, boolean enableCache, boolean reserveDirectory, DirectoryFactory directoryFactory) throws IOException {
super(r);
this.directoryFactory = directoryFactory;
@@ -134,13 +147,8 @@ public class SolrIndexSearcher extends I
// keep the directory from being released while we use it
directoryFactory.incRef(dir);
}
-
- if (dir instanceof FSDirectory) {
- FSDirectory fsDirectory = (FSDirectory) dir;
- indexDir = fsDirectory.getDirectory().getAbsolutePath();
- } else {
- log.warn("WARNING: Directory impl does not support setting indexDir: " + dir.getClass().getName());
- }
+
+ this.indexDir = getIndexDir(dir);
this.closeReader = closeReader;
setSimilarity(schema.getSimilarity());
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/AbstractDistributedZkTestCase.java Sat Jul 7 21:48:19 2012
@@ -33,6 +33,7 @@ import org.apache.solr.servlet.SolrDispa
import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
public abstract class AbstractDistributedZkTestCase extends BaseDistributedSearchTestCase {
@@ -41,6 +42,12 @@ public abstract class AbstractDistribute
protected ZkTestServer zkServer;
private AtomicInteger homeCount = new AtomicInteger();
+ @BeforeClass
+ public static void beforeThisClass() throws Exception {
+ useFactory(null);
+ }
+
+
@Before
@Override
public void setUp() throws Exception {
@@ -55,9 +62,8 @@ public abstract class AbstractDistribute
System.setProperty("zkHost", zkServer.getZkAddress());
System.setProperty("enable.update.log", "true");
System.setProperty("remove.version.field", "true");
- System
- .setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
-
+
+
AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), "solrconfig.xml", "schema.xml");
// set some system properties for use by tests
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Sat Jul 7 21:48:19 2012
@@ -153,14 +153,11 @@ public class FullSolrCloudTest extends A
@BeforeClass
public static void beforeClass() {
- System
- .setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
System.setProperty("solrcloud.update.delay", "0");
}
@AfterClass
public static void afterClass() {
- System.clearProperty("solr.directoryFactory");
System.clearProperty("solrcloud.update.delay");
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Sat Jul 7 21:48:19 2012
@@ -81,12 +81,10 @@ public class TestReplicationHandler exte
// index from previous test method
static int nDocs = 500;
- // TODO: fix this test to not require FSDirectory.. doesnt even work with MockFSDirectory... wtf?
- static String savedFactory;
+
@BeforeClass
public static void beforeClass() throws Exception {
- savedFactory = System.getProperty("solr.DirectoryFactory");
- System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
+ useFactory(null); // need an FS factory
master = new SolrInstance("master", null);
master.setUp();
masterJetty = createJetty(master);
@@ -116,11 +114,6 @@ public class TestReplicationHandler exte
slaveJetty.stop();
master.tearDown();
slave.tearDown();
- if (savedFactory == null) {
- System.clearProperty("solr.directoryFactory");
- } else {
- System.setProperty("solr.directoryFactory", savedFactory);
- }
}
private static JettySolrRunner createJetty(SolrInstance instance) throws Exception {
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedSpellCheckComponentTest.java Sat Jul 7 21:48:19 2012
@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.resp
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SpellingParams;
import org.apache.solr.common.util.NamedList;
+import org.junit.BeforeClass;
/**
* Test for SpellCheckComponent's distributed querying
@@ -47,14 +48,14 @@ public class DistributedSpellCheckCompon
//shardCount=2;
//stress=0;
}
-
- private String saveProp;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ useFactory(null); // need an FS factory
+ }
@Override
public void setUp() throws Exception {
- // this test requires FSDir
- saveProp = System.getProperty("solr.directoryFactory");
- System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
if(random().nextBoolean()) {
requestHandlerName = "spellCheckCompRH";
reqHandlerWithWordbreak = "spellCheckWithWordbreak";
@@ -68,10 +69,6 @@ public class DistributedSpellCheckCompon
@Override
public void tearDown() throws Exception {
super.tearDown();
- if (saveProp == null)
- System.clearProperty("solr.directoryFactory");
- else
- System.setProperty("solr.directoryFactory", saveProp);
}
private void q(Object... q) throws Exception {
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestSolrJ.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestSolrJ.java?rev=1358650&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestSolrJ.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestSolrJ.java Sat Jul 7 21:48:19 2012
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestSolrJ extends SolrTestCaseJ4 {
+
+ public void testSolrJ() {
+ // doCommitPerf();
+ }
+
+ public void doCommitPerf() throws Exception {
+ HttpSolrServer client = new HttpSolrServer("http://localhost:8983/solr");
+
+ long start = System.currentTimeMillis();
+
+ for (int i=0; i<10000; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", Integer.toString(i % 13));
+ client.add(doc);
+ client.commit(true, true, true);
+ }
+
+ long end = System.currentTimeMillis();
+
+ client.shutdown();
+
+ System.out.println("TIME: " + (end-start));
+ }
+
+}
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml Sat Jul 7 21:48:19 2012
@@ -101,17 +101,20 @@
<!-- The DirectoryFactory to use for indexes.
- solr.StandardDirectoryFactory, the default, is filesystem
+ solr.StandardDirectoryFactory is filesystem
based and tries to pick the best implementation for the current
- JVM and platform. One can force a particular implementation
- via solr.MMapDirectoryFactory, solr.NIOFSDirectoryFactory, or
- solr.SimpleFSDirectoryFactory.
+ JVM and platform. solr.NRTCachingDirectoryFactory, the default,
+ wraps solr.StandardDirectoryFactory and caches small files in memory
+ for better NRT performance.
+
+ One can force a particular implementation via solr.MMapDirectoryFactory,
+ solr.NIOFSDirectoryFactory, or solr.SimpleFSDirectoryFactory.
solr.RAMDirectoryFactory is memory based, not
persistent, and doesn't work with replication.
-->
<directoryFactory name="DirectoryFactory"
- class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
<!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Index Config - These settings control low-level behavior of indexing
Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1358650&r1=1358649&r2=1358650&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java Sat Jul 7 21:48:19 2012
@@ -81,8 +81,34 @@ public abstract class SolrTestCaseJ4 ext
resetExceptionIgnores();
endTrackingSearchers();
endTrackingZkClients();
+ resetFactory();
+ resetFactory();
}
+ private static boolean changedFactory = false;
+ private static String savedFactory;
+ /** Use a different directory factory. Passing "null" sets to an FS-based factory */
+ public static void useFactory(String factory) throws Exception {
+ assert !changedFactory;
+ changedFactory = true;
+ savedFactory = System.getProperty("solr.DirectoryFactory");
+ if (factory == null) {
+ factory = random().nextInt(100) < 75 ? "solr.NRTCachingDirectoryFactory" : "solr.StandardDirectoryFactory"; // test the default most of the time
+ }
+ System.setProperty("solr.directoryFactory", factory);
+ }
+
+ private static void resetFactory() throws Exception {
+ if (!changedFactory) return;
+ changedFactory = false;
+ if (savedFactory != null) {
+ System.setProperty("solr.directoryFactory", savedFactory);
+ } else {
+ System.clearProperty("solr.directoryFactory");
+ }
+ }
+
+
@Override
public void setUp() throws Exception {
super.setUp();