You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2018/11/01 05:39:29 UTC
lucene-solr:jira/solr-12730: Adding more logging around when files
are reference counted Added custom CMS impl which sets thread names with MDC
context information
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12730 328b913b4 -> 2bff40112
Adding more logging around when files are reference counted
Added custom CMS impl which sets thread names with MDC context information
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2bff4011
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2bff4011
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2bff4011
Branch: refs/heads/jira/solr-12730
Commit: 2bff40112b91eeec3d3a7b2abfe7d92aa84f5586
Parents: 328b913
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Nov 1 11:09:16 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Nov 1 11:09:16 2018 +0530
----------------------------------------------------------------------
.../apache/lucene/index/IndexFileDeleter.java | 23 ++++++++++++++--
.../org/apache/solr/update/SolrIndexConfig.java | 29 +++++++++++++++++++-
.../processor/DistributedUpdateProcessor.java | 1 -
3 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bff4011/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index f76cb98..cd5cf7f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -20,6 +20,8 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
@@ -564,7 +566,16 @@ final class IndexFileDeleter implements Closeable {
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
if (VERBOSE_REF_COUNTS) {
- infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
+ // todo nocommit remove the extra logging
+ String extra = null;
+ if ("_0.fdt".equals(fileName)) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ new Exception().printStackTrace(writer);
+ extra = stringWriter.toString();
+ }
+
+ infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count + (extra != null ? " and stack is " + extra : ""));
}
}
rc.IncRef();
@@ -602,7 +613,15 @@ final class IndexFileDeleter implements Closeable {
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
if (VERBOSE_REF_COUNTS) {
- infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
+ String extra = null;
+ // todo nocommit remove the extra logging
+ if ("_0.fdt".equals(fileName)) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ new Exception().printStackTrace(writer);
+ extra = stringWriter.toString();
+ }
+ infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count + (extra != null ? " and stack is " + extra : ""));
}
}
if (rc.DecRef() == 0) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bff4011/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index 48b2417..38f16c2 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -18,6 +18,7 @@ package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -25,6 +26,7 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MergePolicy;
@@ -48,6 +50,7 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.SolrPluginUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
import static org.apache.solr.core.Config.assertWarnOrFail;
@@ -283,7 +286,7 @@ public class SolrIndexConfig implements MapSerializable {
private MergeScheduler buildMergeScheduler(IndexSchema schema) {
String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
- MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class);
+ MergeScheduler scheduler = new TestCMS(); // todo nocommit
if (mergeSchedulerInfo != null) {
// LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
@@ -312,4 +315,28 @@ public class SolrIndexConfig implements MapSerializable {
return scheduler;
}
+ static class TestCMS extends ConcurrentMergeScheduler {
+ @Override
+ protected synchronized MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
+ MergeThread mergeThread = super.getMergeThread(writer, merge);
+ final Map<String, String> submitterContext = MDC.getCopyOfContextMap();
+ StringBuilder contextString = new StringBuilder();
+ if (submitterContext != null) {
+ Collection<String> values = submitterContext.values();
+
+ for (String value : values) {
+ contextString.append(value + " ");
+ }
+ if (contextString.length() > 1) {
+ contextString.setLength(contextString.length() - 1);
+ }
+ }
+
+ String ctxStr = contextString.toString().replace("/", "//");
+ final String submitterContextStr = ctxStr.length() <= 512 ? ctxStr : ctxStr.substring(0, 512);
+ mergeThread.setName(mergeThread.getName() + "-processing-" + submitterContextStr);
+ return mergeThread;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2bff4011/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index cc0f8fb..8bdc8bc 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -1388,7 +1388,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
- System.out.println("Doing versionDelete on " + req.getCore().getName() + " dropCmd=" + dropCmd);
}
if (dropCmd) {