You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2020/04/15 13:10:25 UTC
[lucene-solr] branch master updated: LUCENE-9307: Remove the
ability to set the buffer size dynamically on BufferedIndexInput (#1415)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new aa605b3 LUCENE-9307: Remove the ability to set the buffer size dynamically on BufferedIndexInput (#1415)
aa605b3 is described below
commit aa605b3c70fa5a4fa51761f318d134e387059e28
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Wed Apr 15 15:10:11 2020 +0200
LUCENE-9307: Remove the ability to set the buffer size dynamically on BufferedIndexInput (#1415)
---
lucene/CHANGES.txt | 2 +
.../lucene/codecs/MultiLevelSkipListReader.java | 8 +-
.../apache/lucene/store/BufferedIndexInput.java | 23 -----
.../lucene/store/TestBufferedIndexInput.java | 98 ----------------------
4 files changed, 3 insertions(+), 128 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0f18d23..5a1adbe 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -54,6 +54,8 @@ API Changes
must now also implement the default constructor (see MIGRATE.txt).
(Uwe Schindler, Dawid Weiss)
+* LUCENE-9307: BufferedIndexInput#setBufferSize has been removed. (Adrien Grand)
+
Improvements
* LUCENE-8757: When provided with an ExecutorService to run queries across
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
index 517c731..78cceb1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
@@ -21,7 +21,6 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.MathUtil;
@@ -80,8 +79,7 @@ public abstract class MultiLevelSkipListReader implements Closeable {
/** childPointer of last read skip entry with docId <=
* target. */
private long lastChildPointer;
-
- private boolean inputIsBuffered;
+
private final int skipMultiplier;
/** Creates a {@code MultiLevelSkipListReader}. */
@@ -94,7 +92,6 @@ public abstract class MultiLevelSkipListReader implements Closeable {
this.skipInterval = new int[maxSkipLevels];
this.skipMultiplier = skipMultiplier;
this.skipStream [0]= skipStream;
- this.inputIsBuffered = (skipStream instanceof BufferedIndexInput);
this.skipInterval[0] = skipInterval;
for (int i = 1; i < maxSkipLevels; i++) {
// cache skip intervals
@@ -237,9 +234,6 @@ public abstract class MultiLevelSkipListReader implements Closeable {
} else {
// clone this stream, it is already at the start of the current level
skipStream[i] = skipStream[0].clone();
- if (inputIsBuffered && length < BufferedIndexInput.BUFFER_SIZE) {
- ((BufferedIndexInput) skipStream[i]).setBufferSize(Math.max(BufferedIndexInput.MIN_BUFFER_SIZE, (int) length));
- }
// move base stream beyond the current level
skipStream[0].seek(skipStream[0].getFilePointer() + length);
diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
index d0b78ed..cc0ffce 100644
--- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
@@ -73,29 +73,6 @@ public abstract class BufferedIndexInput extends IndexInput implements RandomAcc
this.bufferSize = bufferSize;
}
- /** Change the buffer size used by this IndexInput */
- public final void setBufferSize(int newSize) {
- assert buffer == EMPTY_BYTEBUFFER || bufferSize == buffer.capacity(): "buffer=" + buffer + " bufferSize=" + bufferSize + " buffer.length=" + (buffer != null ? buffer.capacity() : 0);
- if (newSize != bufferSize) {
- checkBufferSize(newSize);
- bufferSize = newSize;
- if (buffer != EMPTY_BYTEBUFFER) {
- // Resize the existing buffer and carefully save as
- // many bytes as possible starting from the current
- // bufferPosition
- ByteBuffer newBuffer = ByteBuffer.allocate(newSize);
- assert newBuffer.order() == ByteOrder.BIG_ENDIAN;
- if (buffer.remaining() > newBuffer.capacity()) {
- buffer.limit(buffer.position() + newBuffer.capacity());
- }
- assert buffer.remaining() <= newBuffer.capacity();
- newBuffer.put(buffer);
- newBuffer.flip();
- buffer = newBuffer;
- }
- }
- }
-
/** Returns buffer size. @see #setBufferSize */
public final int getBufferSize() {
return bufferSize;
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
index b2bc556..905e465 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
@@ -19,23 +19,8 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Random;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.LuceneTestCase;
@@ -199,87 +184,4 @@ public class TestBufferedIndexInput extends LuceneTestCase {
}
}
- public void testSetBufferSize() throws IOException {
- Path indexDir = createTempDir("testSetBufferSize");
- MockFSDirectory dir = new MockFSDirectory(indexDir, random());
- IndexWriter writer = new IndexWriter(
- dir,
- new IndexWriterConfig(new MockAnalyzer(random())).
- setOpenMode(OpenMode.CREATE).
- setMergePolicy(newLogMergePolicy(false))
- );
- for(int i=0;i<37;i++) {
- Document doc = new Document();
- doc.add(newTextField("content", "aaa bbb ccc ddd" + i, Field.Store.YES));
- doc.add(newTextField("id", "" + i, Field.Store.YES));
- writer.addDocument(doc);
- }
-
- dir.allIndexInputs.clear();
-
- IndexReader reader = DirectoryReader.open(writer);
- Term aaa = new Term("content", "aaa");
- Term bbb = new Term("content", "bbb");
-
- reader.close();
-
- dir.tweakBufferSizes();
- writer.deleteDocuments(new Term("id", "0"));
- reader = DirectoryReader.open(writer);
- IndexSearcher searcher = newSearcher(reader);
- ScoreDoc[] hits = searcher.search(new TermQuery(bbb), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(36, hits.length);
-
- reader.close();
-
- dir.tweakBufferSizes();
- writer.deleteDocuments(new Term("id", "4"));
- reader = DirectoryReader.open(writer);
- searcher = newSearcher(reader);
-
- hits = searcher.search(new TermQuery(bbb), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(35, hits.length);
- dir.tweakBufferSizes();
- hits = searcher.search(new TermQuery(new Term("id", "33")), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(1, hits.length);
- hits = searcher.search(new TermQuery(aaa), 1000).scoreDocs;
- dir.tweakBufferSizes();
- assertEquals(35, hits.length);
- writer.close();
- reader.close();
- }
-
- private static class MockFSDirectory extends FilterDirectory {
-
- final List<IndexInput> allIndexInputs = new ArrayList<>();
- final Random rand;
-
- public MockFSDirectory(Path path, Random rand) throws IOException {
- super(new NIOFSDirectory(path));
- this.rand = rand;
- }
-
- public void tweakBufferSizes() {
- //int count = 0;
- for (final IndexInput ip : allIndexInputs) {
- BufferedIndexInput bii = (BufferedIndexInput) ip;
- int bufferSize = 1024 + rand.nextInt(32768);
- bii.setBufferSize(bufferSize);
- //count++;
- }
- //System.out.println("tweak'd " + count + " buffer sizes");
- }
-
- @Override
- public IndexInput openInput(String name, IOContext context) throws IOException {
- // Make random changes to buffer size
- //bufferSize = 1+Math.abs(rand.nextInt() % 10);
- IndexInput f = super.openInput(name, context);
- allIndexInputs.add(f);
- return f;
- }
- }
}