You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2016/09/13 05:20:32 UTC
lucene-solr:master: SOLR-9408: Fix TreeMergeOutputFormat to add
timestamp metadata to commits
Repository: lucene-solr
Updated Branches:
refs/heads/master faf3bc313 -> ef3057e43
SOLR-9408: Fix TreeMergeOutputFormat to add timestamp metadata to commits
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ef3057e4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ef3057e4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ef3057e4
Branch: refs/heads/master
Commit: ef3057e43b6c3783f1324b2893eeb8702c86487c
Parents: faf3bc3
Author: Varun Thacker <va...@apache.org>
Authored: Mon Sep 12 19:11:49 2016 +0530
Committer: Varun Thacker <va...@apache.org>
Committed: Tue Sep 13 10:49:27 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../solr/hadoop/TreeMergeOutputFormat.java | 10 ++++++---
.../solr/update/DirectUpdateHandler2.java | 23 ++++----------------
.../apache/solr/update/SolrIndexSplitter.java | 17 +++------------
.../org/apache/solr/update/SolrIndexWriter.java | 16 ++++++++++++--
5 files changed, 31 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef3057e4/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5a80d06..333a787 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -168,6 +168,9 @@ Bug Fixes
* SOLR-9494: Use of {!collapse} sometimes doesn't correctly return true for Collector.needsScores(), especially when the
query was cached. This can cause an exception when 'q' is a SpanQuery or potentially others. (David Smiley)
+* SOLR-9408: Fix TreeMergeOutputFormat to add timestamp metadata to a commit. SolrCloud replication relies on this.
+ (Jessica Cheng Mallet via Varun Thacker)
+
================== 6.2.0 ==================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef3057e4/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java
----------------------------------------------------------------------
diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java
index 49f844e..e3487ad 100644
--- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java
+++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/TreeMergeOutputFormat.java
@@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
@@ -40,12 +41,11 @@ import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.misc.IndexMergeTool;
import org.apache.lucene.store.Directory;
import org.apache.solr.store.hdfs.HdfsDirectory;
+import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.util.RTimer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-
/**
* See {@link IndexMergeTool}.
*/
@@ -151,7 +151,7 @@ public class TreeMergeOutputFormat extends FileOutputFormat<Text, NullWritable>
LOG.info("Optimizing Solr: forcing tree merge down to {} segments", maxSegments);
timer = new RTimer();
if (maxSegments < Integer.MAX_VALUE) {
- writer.forceMerge(maxSegments);
+ writer.forceMerge(maxSegments);
// TODO: consider perf enhancement for no-deletes merges: bulk-copy the postings data
// see http://lucene.472066.n3.nabble.com/Experience-with-large-merge-factors-tp1637832p1647046.html
}
@@ -161,6 +161,10 @@ public class TreeMergeOutputFormat extends FileOutputFormat<Text, NullWritable>
}
LOG.info("Optimizing Solr: done forcing tree merge down to {} segments in {}ms", maxSegments, timer.getTime());
+ // Set Solr's commit data so the created index is usable by SolrCloud. E.g. Currently SolrCloud relies on
+ // commitTimeMSec in the commit data to do replication.
+ SolrIndexWriter.setCommitData(writer);
+
timer = new RTimer();
LOG.info("Optimizing Solr: Closing index writer");
writer.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef3057e4/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
index 12552cd..258408d 100644
--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
@@ -20,10 +20,8 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Locale;
-import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.LongAdder;
@@ -47,7 +45,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
@@ -516,19 +513,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
return rc;
}
-
- @SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " +
- " but currently suspiciously used for replication as well")
- /*
- Also see SolrIndexSplitter.setCommitData
- */
- private void setCommitData(IndexWriter iw) {
- final Map<String,String> commitData = new HashMap<>();
- commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY,
- String.valueOf(System.currentTimeMillis()));
- iw.setLiveCommitData(commitData.entrySet());
- }
-
+
public void prepareCommit(CommitUpdateCommand cmd) throws IOException {
boolean error=true;
@@ -537,7 +522,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
log.info("start "+cmd);
RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
try {
- setCommitData(iw.get());
+ SolrIndexWriter.setCommitData(iw.get());
iw.get().prepareCommit();
} finally {
iw.decref();
@@ -618,7 +603,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
// SolrCore.verbose("writer.commit() start writer=",writer);
if (writer.hasUncommittedChanges()) {
- setCommitData(writer);
+ SolrIndexWriter.setCommitData(writer);
writer.commit();
} else {
log.info("No uncommitted changes. Skipping IW.commit.");
@@ -803,7 +788,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
}
// todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't be used)
- setCommitData(writer);
+ SolrIndexWriter.setCommitData(writer);
writer.commit();
synchronized (solrCoreState.getUpdateLock()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef3057e4/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
index 14e7063..bd4612e 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
@@ -19,17 +19,15 @@ package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.index.CodecReader;
+import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FilterCodecReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SlowCodecReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -42,7 +40,6 @@ import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.cloud.CompositeIdRouter;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.HashBasedRouter;
-import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.BitsFilteredPostingsEnum;
@@ -140,7 +137,7 @@ public class SolrIndexSplitter {
// we commit explicitly instead of sending a CommitUpdateCommand through the processor chain
// because the sub-shard cores will just ignore such a commit because the update log is not
// in active state at this time.
- setCommitData(iw);
+ SolrIndexWriter.setCommitData(iw);
iw.commit();
success = true;
} finally {
@@ -159,14 +156,6 @@ public class SolrIndexSplitter {
}
- @SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " +
- " but currently suspiciously used for replication as well")
- private void setCommitData(IndexWriter iw) {
- final Map<String,String> commitData = new HashMap<>();
- commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
- iw.setLiveCommitData(commitData.entrySet());
- }
-
FixedBitSet[] split(LeafReaderContext readerContext) throws IOException {
LeafReader reader = readerContext.reader();
FixedBitSet[] docSets = new FixedBitSet[numPieces];
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef3057e4/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
index 8599da1..fcfc559 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
@@ -18,6 +18,8 @@ package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.Codec;
@@ -27,8 +29,9 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.InfoStream;
import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.core.DirectoryFactory.DirContext;
+import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
import org.slf4j.Logger;
@@ -86,7 +89,16 @@ public class SolrIndexWriter extends IndexWriter {
this.directory = directory;
numOpens.incrementAndGet();
}
-
+
+ @SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " +
+ " but currently suspiciously used for replication as well")
+ public static void setCommitData(IndexWriter iw) {
+ log.info("Calling setCommitData with IW:" + iw.toString());
+ final Map<String,String> commitData = new HashMap<>();
+ commitData.put(COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
+ iw.setLiveCommitData(commitData.entrySet());
+ }
+
private void setDirectoryFactory(DirectoryFactory factory) {
this.directoryFactory = factory;
}