You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2014/04/22 23:58:40 UTC
svn commit: r1589298 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/update/
Author: markrmiller
Date: Tue Apr 22 21:58:40 2014
New Revision: 1589298
URL: http://svn.apache.org/r1589298
Log:
SOLR-6002: Fix a couple of ugly issues around SolrIndexWriter close and rollback as well as how SolrIndexWriter manages it's ref counted directory instance.
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1589298&r1=1589297&r2=1589298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Tue Apr 22 21:58:40 2014
@@ -57,6 +57,10 @@ Bug Fixes
* SOLR-5993: ZkController can warn about shard leader conflict even after the conflict
is resolved. (Gregory Chanan via shalin)
+* SOLR-6002: Fix a couple of ugly issues around SolrIndexWriter close and
+ rollback as well as how SolrIndexWriter manages it's ref counted directory
+ instance. (Mark Miller)
+
Other Changes
---------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1589298&r1=1589297&r2=1589298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java Tue Apr 22 21:58:40 2014
@@ -173,6 +173,7 @@ public abstract class CachingDirectoryFa
this.getClass().getSimpleName(), val);
try {
// if there are still refs out, we have to wait for them
+ assert val.refCnt > -1 : val.refCnt;
int cnt = 0;
while(val.refCnt != 0) {
wait(100);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1589298&r1=1589297&r2=1589298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java Tue Apr 22 21:58:40 2014
@@ -17,7 +17,6 @@
package org.apache.solr.core;
-import org.apache.commons.io.IOUtils;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexDeletionPolicy;
@@ -88,6 +87,7 @@ import org.apache.solr.update.processor.
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.IOUtils;
import org.apache.solr.util.PropertiesInputStream;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -417,8 +417,8 @@ public final class SolrCore implements S
ParserConfigurationException, SAXException {
solrCoreState.increfSolrCoreState();
-
- if (!getNewIndexDir().equals(getIndexDir())) {
+ boolean indexDirChange = !getNewIndexDir().equals(getIndexDir());
+ if (indexDirChange || !coreConfig.getSolrConfig().nrtMode) {
// the directory is changing, don't pass on state
prev = null;
}
@@ -427,6 +427,8 @@ public final class SolrCore implements S
coreConfig.getIndexSchema(), coreDescriptor, updateHandler, this.solrDelPolicy, prev);
core.solrDelPolicy = this.solrDelPolicy;
+
+ // we open a new indexwriter to pick up the latest config
core.getUpdateHandler().getSolrCoreState().newIndexWriter(core, false);
core.getSearcher(true, false, null, true);
@@ -853,7 +855,15 @@ public final class SolrCore implements S
if (e instanceof OutOfMemoryError) {
throw (OutOfMemoryError)e;
}
- close();
+
+ try {
+ this.close();
+ } catch (Throwable t) {
+ if (t instanceof OutOfMemoryError) {
+ throw (OutOfMemoryError)t;
+ }
+ log.error("Error while closing", t);
+ }
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
e.getMessage(), e);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1589298&r1=1589297&r2=1589298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java Tue Apr 22 21:58:40 2014
@@ -22,6 +22,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -29,6 +30,7 @@ import org.apache.solr.core.CoreContaine
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.IOUtils;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,7 +70,6 @@ public final class DefaultSolrCoreState
closer.closeWriter(indexWriter);
} else if (indexWriter != null) {
log.info("closing IndexWriter...");
- indexWriter.close();
}
indexWriter = null;
} catch (Exception e) {
@@ -159,21 +160,9 @@ public final class DefaultSolrCoreState
try {
if (indexWriter != null) {
if (!rollback) {
- try {
- log.info("Closing old IndexWriter... core=" + coreName);
- indexWriter.close();
- } catch (Exception e) {
- SolrException.log(log, "Error closing old IndexWriter. core="
- + coreName, e);
- }
+ closeIndexWriter(coreName);
} else {
- try {
- log.info("Rollback old IndexWriter... core=" + coreName);
- indexWriter.rollback();
- } catch (Exception e) {
- SolrException.log(log, "Error rolling back old IndexWriter. core="
- + coreName, e);
- }
+ rollbackIndexWriter(coreName);
}
}
indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2");
@@ -187,6 +176,41 @@ public final class DefaultSolrCoreState
}
}
}
+
+ private void closeIndexWriter(String coreName) {
+ try {
+ log.info("Closing old IndexWriter... core=" + coreName);
+ Directory dir = indexWriter.getDirectory();
+ try {
+ IOUtils.closeQuietly(indexWriter);
+ } finally {
+ if (IndexWriter.isLocked(dir)) {
+ IndexWriter.unlock(dir);
+ }
+ }
+ } catch (Exception e) {
+ SolrException.log(log, "Error closing old IndexWriter. core="
+ + coreName, e);
+ }
+ }
+
+ private void rollbackIndexWriter(String coreName) {
+ try {
+ log.info("Rollback old IndexWriter... core=" + coreName);
+ Directory dir = indexWriter.getDirectory();
+ try {
+
+ indexWriter.rollback();
+ } finally {
+ if (IndexWriter.isLocked(dir)) {
+ IndexWriter.unlock(dir);
+ }
+ }
+ } catch (Exception e) {
+ SolrException.log(log, "Error rolling back old IndexWriter. core="
+ + coreName, e);
+ }
+ }
@Override
public synchronized void closeIndexWriter(SolrCore core, boolean rollback)
@@ -217,21 +241,9 @@ public final class DefaultSolrCoreState
if (indexWriter != null) {
if (!rollback) {
- try {
- log.info("Closing old IndexWriter... core=" + coreName);
- indexWriter.close();
- } catch (Exception e) {
- SolrException.log(log, "Error closing old IndexWriter. core="
- + coreName, e);
- }
+ closeIndexWriter(coreName);
} else {
- try {
- log.info("Rollback old IndexWriter... core=" + coreName);
- indexWriter.rollback();
- } catch (Exception e) {
- SolrException.log(log, "Error rolling back old IndexWriter. core="
- + coreName, e);
- }
+ rollbackIndexWriter(coreName);
}
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1589298&r1=1589297&r2=1589298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Tue Apr 22 21:58:40 2014
@@ -745,7 +745,7 @@ public class DirectUpdateHandler2 extend
try {
if (tryToCommit) {
-
+ log.info("Committing on IndexWriter close.");
CommitUpdateCommand cmd = new CommitUpdateCommand(req, false);
cmd.openSearcher = false;
cmd.waitSearcher = false;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1589298&r1=1589297&r2=1589298&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Tue Apr 22 21:58:40 2014
@@ -17,10 +17,7 @@
package org.apache.solr.update;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.PrintStream;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.Codec;
@@ -29,11 +26,10 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.PrintStreamInfoStream;
-import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,13 +44,17 @@ public class SolrIndexWriter extends Ind
// These should *only* be used for debugging or monitoring purposes
public static final AtomicLong numOpens = new AtomicLong();
public static final AtomicLong numCloses = new AtomicLong();
-
+
/** Stored into each Lucene commit to record the
* System.currentTimeMillis() when commit was called. */
public static final String COMMIT_TIME_MSEC_KEY = "commitTimeMSec";
+ private static final Object CLOSE_LOCK = new Object();
+
String name;
private DirectoryFactory directoryFactory;
+ private InfoStream infoStream;
+ private Directory directory;
public static SolrIndexWriter create(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
@@ -81,6 +81,8 @@ public class SolrIndexWriter extends Ind
);
log.debug("Opened Writer " + name);
this.name = name;
+ infoStream = getConfig().getInfoStream();
+ this.directory = directory;
numOpens.incrementAndGet();
}
@@ -120,62 +122,54 @@ public class SolrIndexWriter extends Ind
*/
private volatile boolean isClosed = false;
-
@Override
public void close() throws IOException {
log.debug("Closing Writer " + name);
- Directory directory = getDirectory();
- final InfoStream infoStream = isClosed ? null : getConfig().getInfoStream();
try {
- while (true) {
- try {
- super.close();
- } catch (ThreadInterruptedException e) {
- // don't allow interruption
- continue;
- } catch (Throwable t) {
- if (t instanceof OutOfMemoryError) {
- throw (OutOfMemoryError) t;
- }
- log.error("Error closing IndexWriter, trying rollback", t);
- super.rollback();
- }
- if (IndexWriter.isLocked(directory)) {
- try {
- IndexWriter.unlock(directory);
- } catch (Exception e) {
- log.error("Coud not unlock directory after seemingly failed IndexWriter#close()", e);
- }
- }
- break;
+ super.close();
+ } catch (Throwable t) {
+ if (t instanceof OutOfMemoryError) {
+ throw (OutOfMemoryError) t;
}
+ log.error("Error closing IndexWriter", t);
} finally {
- if (infoStream != null) {
- infoStream.close();
- }
- isClosed = true;
- directoryFactory.release(directory);
- numCloses.incrementAndGet();
+ cleanup();
}
}
@Override
public void rollback() throws IOException {
- Directory dir = getDirectory();
+ log.debug("Rollback Writer " + name);
try {
- while (true) {
- try {
- super.rollback();
- } catch (ThreadInterruptedException e) {
- // don't allow interruption
- continue;
- }
- break;
+ super.rollback();
+ } catch (Throwable t) {
+ if (t instanceof OutOfMemoryError) {
+ throw (OutOfMemoryError) t;
}
+ log.error("Exception rolling back IndexWriter", t);
} finally {
- isClosed = true;
- directoryFactory.release(dir);
+ cleanup();
+ }
+ }
+
+ private void cleanup() throws IOException {
+ // It's kind of an implementation detail whether
+ // or not IndexWriter#close calls rollback, so
+ // we assume it may or may not
+ boolean doClose = false;
+ synchronized (CLOSE_LOCK) {
+ if (!isClosed) {
+ doClose = true;
+ isClosed = true;
+ }
+ }
+ if (doClose) {
+
+ if (infoStream != null) {
+ IOUtils.closeQuietly(infoStream);
+ }
numCloses.incrementAndGet();
+ directoryFactory.release(directory);
}
}