You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/02/16 02:29:21 UTC
svn commit: r1446834 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/codecs/ lucene/core/
lucene/core/src/test/org/apache/l...
Author: markrmiller
Date: Sat Feb 16 01:29:19 2013
New Revision: 1446834
URL: http://svn.apache.org/r1446834
Log:
SOLR-4459: The Replication 'index move' rather than copy optimization doesn't kick in when using NRTCachingDirectory or the rate limiting feature.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1446834&r1=1446833&r2=1446834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Sat Feb 16 01:29:19 2013
@@ -83,7 +83,7 @@ Bug Fixes
* SOLR-3926: Solr should support better way of finding active sorts (Eirik Lygre via
Erick Erickson)
-* SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
+* SOLR-4342: Fix DataImportHandler stats to be a proper Map (hossman)
* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
@@ -101,6 +101,10 @@ Bug Fixes
* SOLR-4463: Fix SolrCoreState reference counting. (Mark Miller)
+* SOLR-4459: The Replication 'index move' rather than copy optimization doesn't
+ kick in when using NRTCachingDirectory or the rate limiting feature.
+ (Mark Miller)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1446834&r1=1446833&r2=1446834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Sat Feb 16 01:29:19 2013
@@ -103,6 +103,10 @@ public abstract class DirectoryFactory i
/**
* Override for more efficient moves.
*
+ * Intended for use with replication - use
+ * carefully - some Directory wrappers will
+ * cache files for example.
+ *
* @throws IOException If there is a low-level I/O error.
*/
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java?rev=1446834&r1=1446833&r2=1446834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java Sat Feb 16 01:29:19 2013
@@ -23,6 +23,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.NRTCachingDirectory;
+import org.apache.lucene.store.RateLimitedDirectoryWrapper;
/**
* Directory provider which mimics original Solr
@@ -69,15 +71,29 @@ public class StandardDirectoryFactory ex
/**
* Override for more efficient moves.
*
+ * Intended for use with replication - use
+ * carefully - some Directory wrappers will
+ * cache files for example.
+ *
+ * This implementation works with two wrappers:
+ * NRTCachingDirectory and RateLimitedDirectoryWrapper.
+ *
+ * You should first {@link Directory#sync(java.util.Collection)} any file that will be
+ * moved or avoid cached files through settings.
+ *
* @throws IOException
* If there is a low-level I/O error.
*/
@Override
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
throws IOException {
- if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) {
- File dir1 = ((FSDirectory) fromDir).getDirectory();
- File dir2 = ((FSDirectory) toDir).getDirectory();
+
+ Directory baseFromDir = getBaseDir(fromDir);
+ Directory baseToDir = getBaseDir(fromDir);
+
+ if (baseFromDir instanceof FSDirectory && baseToDir instanceof FSDirectory) {
+ File dir1 = ((FSDirectory) baseFromDir).getDirectory();
+ File dir2 = ((FSDirectory) baseToDir).getDirectory();
File indexFileInTmpDir = new File(dir1, fileName);
File indexFileInIndex = new File(dir2, fileName);
boolean success = indexFileInTmpDir.renameTo(indexFileInIndex);
@@ -89,4 +105,18 @@ public class StandardDirectoryFactory ex
super.move(fromDir, toDir, fileName, ioContext);
}
+ // special hack to work with NRTCachingDirectory and RateLimitedDirectoryWrapper
+ private Directory getBaseDir(Directory dir) {
+ Directory baseDir;
+ if (dir instanceof NRTCachingDirectory) {
+ baseDir = ((NRTCachingDirectory)dir).getDelegate();
+ } else if (dir instanceof RateLimitedDirectoryWrapper) {
+ baseDir = ((RateLimitedDirectoryWrapper)dir).getDelegate();
+ } else {
+ baseDir = dir;
+ }
+
+ return baseDir;
+ }
+
}