You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/06/05 09:51:05 UTC
[04/19] lucene-solr:master: cutover RandomIndexWriter to return
sequence numbers
cutover RandomIndexWriter to return sequence numbers
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/38e9822c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/38e9822c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/38e9822c
Branch: refs/heads/master
Commit: 38e9822cb4bc49fe0516ee8ceb24ff2678192471
Parents: c9aaa77
Author: Mike McCandless <mi...@apache.org>
Authored: Tue May 24 20:22:47 2016 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue May 24 20:22:47 2016 -0400
----------------------------------------------------------------------
.../apache/lucene/index/DocumentsWriter.java | 10 ++-
.../org/apache/lucene/index/IndexWriter.java | 31 +++++++---
.../org/apache/lucene/index/TwoPhaseCommit.java | 2 -
.../lucene/index/TestTwoPhaseCommitTool.java | 6 +-
.../directory/DirectoryTaxonomyWriter.java | 8 +--
.../apache/lucene/index/RandomIndexWriter.java | 64 +++++++++++---------
6 files changed, 73 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38e9822c/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
index 6b698db..9f1bdd3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -161,11 +161,15 @@ final class DocumentsWriter implements Closeable, Accountable {
return seqNo;
}
- synchronized boolean updateDocValues(DocValuesUpdate... updates) throws IOException {
+ synchronized long updateDocValues(DocValuesUpdate... updates) throws IOException {
final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
- deleteQueue.addDocValuesUpdates(updates);
+ long seqNo = deleteQueue.addDocValuesUpdates(updates);
flushControl.doOnDelete();
- return applyAllDeletes(deleteQueue);
+ if (applyAllDeletes(deleteQueue)) {
+ seqNo = -seqNo;
+ }
+
+ return seqNo;
}
DocumentsWriterDeleteQueue currentDeleteSession() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38e9822c/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 4791a15..9a520b1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1571,17 +1571,23 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* @throws IOException
* if there is a low-level IO error
*/
- public void updateNumericDocValue(Term term, String field, long value) throws IOException {
+ public long updateNumericDocValue(Term term, String field, long value) throws IOException {
ensureOpen();
if (!globalFieldNumberMap.contains(field, DocValuesType.NUMERIC)) {
throw new IllegalArgumentException("can only update existing numeric-docvalues fields!");
}
try {
- if (docWriter.updateDocValues(new NumericDocValuesUpdate(term, field, value))) {
+ long seqNo = docWriter.updateDocValues(new NumericDocValuesUpdate(term, field, value));
+ if (seqNo < 0) {
+ seqNo = -seqNo;
processEvents(true, false);
}
+ return seqNo;
} catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "updateNumericDocValue");
+
+ // dead code but javac disagrees:
+ return -1;
}
}
@@ -1605,7 +1611,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* @throws IOException
* if there is a low-level IO error
*/
- public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
+ public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
ensureOpen();
if (value == null) {
throw new IllegalArgumentException("cannot update a field to a null value: " + field);
@@ -1614,11 +1620,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
throw new IllegalArgumentException("can only update existing binary-docvalues fields!");
}
try {
- if (docWriter.updateDocValues(new BinaryDocValuesUpdate(term, field, value))) {
+ long seqNo = docWriter.updateDocValues(new BinaryDocValuesUpdate(term, field, value));
+ if (seqNo < 0) {
+ seqNo = -seqNo;
processEvents(true, false);
}
+ return seqNo;
} catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "updateBinaryDocValue");
+
+ // dead code but javac disagrees:
+ return -1;
}
}
@@ -1635,7 +1647,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* @throws IOException
* if there is a low-level IO error
*/
- public void updateDocValues(Term term, Field... updates) throws IOException {
+ public long updateDocValues(Term term, Field... updates) throws IOException {
ensureOpen();
DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length];
for (int i = 0; i < updates.length; i++) {
@@ -1662,11 +1674,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
}
try {
- if (docWriter.updateDocValues(dvUpdates)) {
+ long seqNo = docWriter.updateDocValues(dvUpdates);
+ if (seqNo < 0) {
+ seqNo = -seqNo;
processEvents(true, false);
}
+ return seqNo;
} catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "updateDocValues");
+
+ // dead code but javac disagrees:
+ return -1;
}
}
@@ -3045,7 +3063,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
@Override
public final long commit() throws IOException {
ensureOpen();
- // nocommit should we put seq no into sis?
return commitInternal(config.getMergePolicy());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38e9822c/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java b/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java
index ffa284e..c4ba78c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TwoPhaseCommit.java
@@ -50,7 +50,5 @@ public interface TwoPhaseCommit {
* {@link #prepareCommit()}, this method is used to roll all other objects
* back to their previous state.
*/
- // nocommit return long?
public void rollback() throws IOException;
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38e9822c/lucene/core/src/test/org/apache/lucene/index/TestTwoPhaseCommitTool.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTwoPhaseCommitTool.java b/lucene/core/src/test/org/apache/lucene/index/TestTwoPhaseCommitTool.java
index def90f2..9ef5a30 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTwoPhaseCommitTool.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTwoPhaseCommitTool.java
@@ -50,8 +50,7 @@ public class TestTwoPhaseCommitTool extends LuceneTestCase {
if (failOnPrepare) {
throw new IOException("failOnPrepare");
}
- // nocommit hmm
- return -1;
+ return 1;
}
@Override
@@ -65,8 +64,7 @@ public class TestTwoPhaseCommitTool extends LuceneTestCase {
if (failOnCommit) {
throw new RuntimeException("failOnCommit");
}
- // nocommit hmm
- return -1;
+ return 1;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38e9822c/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
index 74d77e2..97acb49 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
@@ -581,14 +581,14 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter {
}
@Override
- public synchronized void commit() throws IOException {
+ public synchronized long commit() throws IOException {
ensureOpen();
// LUCENE-4972: if we always call setCommitData, we create empty commits
String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH);
if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) {
indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
}
- indexWriter.commit();
+ return indexWriter.commit();
}
/** Combine original user data with the taxonomy epoch. */
@@ -616,14 +616,14 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter {
* See {@link IndexWriter#prepareCommit}.
*/
@Override
- public synchronized void prepareCommit() throws IOException {
+ public synchronized long prepareCommit() throws IOException {
ensureOpen();
// LUCENE-4972: if we always call setCommitData, we create empty commits
String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH);
if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) {
indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
}
- indexWriter.prepareCommit();
+ return indexWriter.prepareCommit();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38e9822c/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
index 0f67882..d46c248 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
@@ -130,14 +130,15 @@ public class RandomIndexWriter implements Closeable {
* Adds a Document.
* @see IndexWriter#addDocument(Iterable)
*/
- public <T extends IndexableField> void addDocument(final Iterable<T> doc) throws IOException {
+ public <T extends IndexableField> long addDocument(final Iterable<T> doc) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
+ long seqNo;
if (r.nextInt(5) == 3) {
// TODO: maybe, we should simply buffer up added docs
// (but we need to clone them), and only when
// getReader, commit, etc. are called, we do an
// addDocuments? Would be better testing.
- w.addDocuments(new Iterable<Iterable<T>>() {
+ seqNo = w.addDocuments(new Iterable<Iterable<T>>() {
@Override
public Iterator<Iterable<T>> iterator() {
@@ -167,10 +168,12 @@ public class RandomIndexWriter implements Closeable {
}
});
} else {
- w.addDocument(doc);
+ seqNo = w.addDocument(doc);
}
maybeFlushOrCommit();
+
+ return seqNo;
}
private void maybeFlushOrCommit() throws IOException {
@@ -195,26 +198,29 @@ public class RandomIndexWriter implements Closeable {
}
}
- public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+ public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.addDocuments(docs);
+ long seqNo = w.addDocuments(docs);
maybeFlushOrCommit();
+ return seqNo;
}
- public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
+ public long updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.updateDocuments(delTerm, docs);
+ long seqNo = w.updateDocuments(delTerm, docs);
maybeFlushOrCommit();
+ return seqNo;
}
/**
* Updates a document.
* @see IndexWriter#updateDocument(Term, Iterable)
*/
- public <T extends IndexableField> void updateDocument(Term t, final Iterable<T> doc) throws IOException {
+ public <T extends IndexableField> long updateDocument(Term t, final Iterable<T> doc) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
+ long seqNo;
if (r.nextInt(5) == 3) {
- w.updateDocuments(t, new Iterable<Iterable<T>>() {
+ seqNo = w.updateDocuments(t, new Iterable<Iterable<T>>() {
@Override
public Iterator<Iterable<T>> iterator() {
@@ -243,49 +249,51 @@ public class RandomIndexWriter implements Closeable {
}
});
} else {
- w.updateDocument(t, doc);
+ seqNo = w.updateDocument(t, doc);
}
maybeFlushOrCommit();
+
+ return seqNo;
}
- public void addIndexes(Directory... dirs) throws IOException {
+ public long addIndexes(Directory... dirs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.addIndexes(dirs);
+ return w.addIndexes(dirs);
}
- public void addIndexes(CodecReader... readers) throws IOException {
+ public long addIndexes(CodecReader... readers) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.addIndexes(readers);
+ return w.addIndexes(readers);
}
- public void updateNumericDocValue(Term term, String field, Long value) throws IOException {
+ public long updateNumericDocValue(Term term, String field, Long value) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.updateNumericDocValue(term, field, value);
+ return w.updateNumericDocValue(term, field, value);
}
- public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
+ public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.updateBinaryDocValue(term, field, value);
+ return w.updateBinaryDocValue(term, field, value);
}
- public void updateDocValues(Term term, Field... updates) throws IOException {
+ public long updateDocValues(Term term, Field... updates) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.updateDocValues(term, updates);
+ return w.updateDocValues(term, updates);
}
- public void deleteDocuments(Term term) throws IOException {
+ public long deleteDocuments(Term term) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.deleteDocuments(term);
+ return w.deleteDocuments(term);
}
- public void deleteDocuments(Query q) throws IOException {
+ public long deleteDocuments(Query q) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.deleteDocuments(q);
+ return w.deleteDocuments(q);
}
- public void commit() throws IOException {
+ public long commit() throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
- w.commit();
+ return w.commit();
}
public int numDocs() {
@@ -296,8 +304,8 @@ public class RandomIndexWriter implements Closeable {
return w.maxDoc();
}
- public void deleteAll() throws IOException {
- w.deleteAll();
+ public long deleteAll() throws IOException {
+ return w.deleteAll();
}
public DirectoryReader getReader() throws IOException {