You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2013/03/21 23:51:04 UTC
svn commit: r1459567 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/
lucene/benchmark/ lucene/classification/ lucene/classification/src/
lucene/code...
Author: markrmiller
Date: Thu Mar 21 22:51:02 2013
New Revision: 1459567
URL: http://svn.apache.org/r1459567
Log:
SOLR-4624: CachingDirectoryFactory does not need to support forceNew any longer and it appears to be causing a missing close directory bug. forceNew is no longer respected and will be removed in 4.3.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ (props changed)
lucene/dev/branches/branch_4x/lucene/classification/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/ivy.xml (props changed)
lucene/dev/branches/branch_4x/lucene/classification/src/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortRandom.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestTotalHitCountCollector.java (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestConfig.java (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Thu Mar 21 22:51:02 2013
@@ -78,6 +78,8 @@ Other Changes
* SOLR-4607: Use noggit 0.5 release jar rather than a forked copy. (Yonik Seeley, Robert Muir)
+* SOLR-4624: forceNew has been removed from the DirectoryFactory and related java apis.
+
================== 4.2.1 ==================
Versions of Major Components
@@ -195,6 +197,9 @@ Bug Fixes
values and top-level phrase slops on queries produced by nested
sub-parsers. (yonik)
+* SOLR-4624: CachingDirectoryFactory does not need to support forceNew any
+ longer and it appears to be causing a missing close directory bug. forceNew
+ is no longer respected and will be removed in 4.3. (Mark Miller)
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Thu Mar 21 22:51:02 2013
@@ -63,10 +63,6 @@ public abstract class CachingDirectoryFa
this.closeEntries.add(this);
}
public int refCnt = 1;
- // if we are latestForPath, I'm currently using my path
- // otherwise a new Directory instance is using my path
- // and I must be manipulated by Directory
- public boolean latestForPath = false;
// has close(Directory) been called on this?
public boolean closeDirectoryCalled = false;
public boolean doneWithDir = false;
@@ -218,12 +214,8 @@ public abstract class CachingDirectoryFa
byDirectoryCache.remove(directory);
- // if it's been closed, it's path is now
- // owned by another Directory instance
- if (!cacheValue.latestForPath) {
- byPathCache.remove(cacheValue.path);
- cacheValue.latestForPath = true;
- }
+ byPathCache.remove(cacheValue.path);
+
}
}
}
@@ -314,22 +306,10 @@ public abstract class CachingDirectoryFa
* (non-Javadoc)
*
* @see org.apache.solr.core.DirectoryFactory#get(java.lang.String,
- * java.lang.String)
- */
- @Override
- public final Directory get(String path, DirContext dirContext, String rawLockType)
- throws IOException {
- return get(path, dirContext, rawLockType, false);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.solr.core.DirectoryFactory#get(java.lang.String,
* java.lang.String, boolean)
*/
@Override
- public final Directory get(String path, DirContext dirContext, String rawLockType, boolean forceNew)
+ public final Directory get(String path, DirContext dirContext, String rawLockType)
throws IOException {
String fullPath = normalize(path);
synchronized (this) {
@@ -341,24 +321,9 @@ public abstract class CachingDirectoryFa
Directory directory = null;
if (cacheValue != null) {
directory = cacheValue.directory;
- if (forceNew) {
- cacheValue.doneWithDir = true;
-
- // we make a quick close attempt,
- // otherwise this should be closed
- // when whatever is using it, releases it
- if (cacheValue.refCnt == 0) {
- closeDirectory(cacheValue);
- }
-
- // close the entry, it will be owned by the new dir
- // we count on it being released by directory
- cacheValue.latestForPath = true;
-
- }
}
- if (directory == null || forceNew) {
+ if (directory == null) {
directory = create(fullPath, dirContext);
directory = rateLimit(directory);
@@ -369,7 +334,7 @@ public abstract class CachingDirectoryFa
byDirectoryCache.put(directory, newCacheValue);
byPathCache.put(fullPath, newCacheValue);
- log.info("return new directory for " + fullPath + " forceNew: " + forceNew);
+ log.info("return new directory for " + fullPath);
} else {
cacheValue.refCnt++;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java Thu Mar 21 22:51:02 2013
@@ -140,9 +140,6 @@ public abstract class DirectoryFactory i
* Returns the Directory for a given path, using the specified rawLockType.
* Will return the same Directory instance for the same path.
*
- * Note: sometimes you might pass null for the rawLockType when
- * you know the Directory exists and the rawLockType is already
- * in use.
*
* @throws IOException If there is a low-level I/O error.
*/
@@ -150,18 +147,6 @@ public abstract class DirectoryFactory i
throws IOException;
/**
- * Returns the Directory for a given path, using the specified rawLockType.
- * Will return the same Directory instance for the same path unless forceNew,
- * in which case a new Directory is returned. There is no need to call
- * {@link #doneWithDirectory(Directory)} in this case - the old Directory
- * will be closed when it's ref count hits 0.
- *
- * @throws IOException If there is a low-level I/O error.
- */
- public abstract Directory get(String path, DirContext dirContext, String rawLockType,
- boolean forceNew) throws IOException;
-
- /**
* Increment the number of references to the given Directory. You must call
* release for every call to this method.
*
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java Thu Mar 21 22:51:02 2013
@@ -497,7 +497,7 @@ public final class SolrCore implements S
log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist."
+ " Creating new index...");
- SolrIndexWriter writer = SolrIndexWriter.create("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.indexConfig, solrDelPolicy, codec, false);
+ SolrIndexWriter writer = SolrIndexWriter.create("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.indexConfig, solrDelPolicy, codec);
writer.close();
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Thu Mar 21 22:51:02 2013
@@ -104,7 +104,7 @@ public final class DefaultSolrCoreState
}
if (indexWriter == null) {
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false);
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2");
}
initRefCntWriter();
writerFree = false;
@@ -172,7 +172,7 @@ public final class DefaultSolrCoreState
}
}
}
- indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", forceNewDir);
+ indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2");
log.info("New IndexWriter is ready to be used.");
// we need to null this so it picks up the new writer next get call
refCntWriter = null;
@@ -189,10 +189,10 @@ public final class DefaultSolrCoreState
newIndexWriter(core, true, false);
}
- protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name, boolean forceNewDirectory) throws IOException {
+ protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name) throws IOException {
return SolrIndexWriter.create(name, core.getNewIndexDir(),
core.getDirectoryFactory(), false, core.getSchema(),
- core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory);
+ core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
}
@Override
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Thu Mar 21 22:51:02 2013
@@ -102,7 +102,7 @@ public class SolrIndexSplitter {
String path = paths.get(partitionNumber);
iw = SolrIndexWriter.create("SplittingIndexWriter"+partitionNumber + " " + ranges.get(partitionNumber), path,
core.getDirectoryFactory(), true, core.getSchema(),
- core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec(), true);
+ core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
}
try {
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Thu Mar 21 22:51:02 2013
@@ -56,10 +56,10 @@ public class SolrIndexWriter extends Ind
String name;
private DirectoryFactory directoryFactory;
- public static SolrIndexWriter create(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException {
+ public static SolrIndexWriter create(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
SolrIndexWriter w = null;
- final Directory d = directoryFactory.get(path, DirContext.DEFAULT, config.lockType, forceNewDirectory);
+ final Directory d = directoryFactory.get(path, DirContext.DEFAULT, config.lockType);
try {
w = new SolrIndexWriter(name, path, d, create, schema,
config, delPolicy, codec);
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java?rev=1459567&r1=1459566&r2=1459567&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java Thu Mar 21 22:51:02 2013
@@ -32,7 +32,6 @@ import org.junit.Test;
public class CachingDirectoryFactoryTest extends SolrTestCaseJ4 {
private Map<String,Tracker> dirs = new HashMap<String,Tracker>();
- private List<Tracker> oldDirs = new ArrayList<Tracker>();
private volatile boolean stop = false;
private class Tracker {
@@ -96,16 +95,6 @@ public class CachingDirectoryFactoryTest
}
}
}
- sz = oldDirs.size();
- if (sz > 0) {
- for (Tracker tracker : oldDirs) {
- int cnt = tracker.refCnt.get();
- for (int i = 0; i < cnt; i++) {
- tracker.refCnt.decrementAndGet();
- df.release(tracker.dir);
- }
- }
- }
}
@@ -187,18 +176,7 @@ public class CachingDirectoryFactoryTest
tracker.dir = df.get(path, DirContext.DEFAULT, null);
dirs.put(path, tracker);
} else {
- if (random.nextInt(10) > 6) {
- Tracker oldTracker = new Tracker();
- oldTracker.refCnt = new AtomicInteger(tracker.refCnt.get());
- oldTracker.path = tracker.path;
- oldTracker.dir = tracker.dir;
- oldDirs.add(oldTracker);
-
- tracker.dir = df.get(path, DirContext.DEFAULT, null, true);
- tracker.refCnt = new AtomicInteger(0);
- } else {
- tracker.dir = df.get(path, DirContext.DEFAULT, null);
- }
+ tracker.dir = df.get(path, DirContext.DEFAULT, null);
}
tracker.refCnt.incrementAndGet();
}