You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/02/01 14:02:49 UTC
[lucene] branch branch_9x updated (6711a805e43 -> d1e4a3a44d8)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from 6711a805e43 Adjust return type for VectorUtil methods (#12122)
new 6df5f0e1709 GITHUB-11868: Add FilterIndexInput and FilterIndexOutput wrapper classes (#11958)
new d1e4a3a44d8 Reduce bloom filter size by using the optimal count for hash functions. (#11900)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 9 +-
.../store/EndiannessReverserIndexInput.java | 90 +-------------
.../store/EndiannessReverserIndexOutput.java | 67 +----------
.../store/EndiannessReverserUtil.java | 4 +-
.../lucene/codecs/bloom/BloomFilterFactory.java | 4 +-
.../codecs/bloom/BloomFilteringPostingsFormat.java | 13 ++-
.../codecs/bloom/DefaultBloomFilterFactory.java | 2 +-
.../org/apache/lucene/codecs/bloom/FuzzySet.java | 129 +++++++++++----------
.../apache/lucene/codecs/bloom/HashFunction.java | 2 +-
.../apache/lucene/codecs/bloom/MurmurHash2.java | 101 ----------------
.../apache/lucene/codecs/bloom/MurmurHash64.java | 85 ++++++++++++++
.../lucene/store/ByteBuffersIndexOutput.java | 4 +
.../org/apache/lucene/store/FilterIndexInput.java} | 75 ++++++------
.../org/apache/lucene/store/FilterIndexOutput.java | 84 ++++++++++++++
.../lucene/store/RateLimitedIndexOutput.java | 33 ++----
.../apache/lucene/index/TestFilterIndexInput.java | 95 +++++++++++++++
.../org/apache/lucene/index/TestIndexInput.java | 22 ++--
.../apache/lucene/index/TestLazyProxSkipping.java | 41 +------
.../lucene/index/TestMultiLevelSkipList.java | 35 ++----
.../apache/lucene/store/TestFilterIndexOutput.java | 67 +++++++++++
.../org/apache/lucene/facet/SlowDirectory.java | 74 +++---------
.../lucene/misc/store/ByteTrackingIndexOutput.java | 61 ++--------
.../tests/index/BaseIndexFileFormatTestCase.java | 27 +----
.../lucene/tests/store/CorruptingIndexOutput.java | 22 +---
.../lucene/tests/store/MockIndexInputWrapper.java | 51 ++++----
.../lucene/tests/store/MockIndexOutputWrapper.java | 43 +++----
.../lucene/tests/util/ThrottledIndexOutput.java | 48 +++-----
27 files changed, 582 insertions(+), 706 deletions(-)
delete mode 100644 lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java
create mode 100644 lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.java
copy lucene/{backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserChecksumIndexInput.java => core/src/java/org/apache/lucene/store/FilterIndexInput.java} (51%)
create mode 100644 lucene/core/src/java/org/apache/lucene/store/FilterIndexOutput.java
create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestFilterIndexInput.java
create mode 100644 lucene/core/src/test/org/apache/lucene/store/TestFilterIndexOutput.java
[lucene] 02/02: Reduce bloom filter size by using the optimal count for hash functions. (#11900)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit d1e4a3a44d8918262def84ae3363857772ea74b7
Author: Jean-François BOEUF <je...@gmail.com>
AuthorDate: Wed Feb 1 14:35:50 2023 +0100
Reduce bloom filter size by using the optimal count for hash functions. (#11900)
---
lucene/CHANGES.txt | 5 +-
.../lucene/codecs/bloom/BloomFilterFactory.java | 4 +-
.../codecs/bloom/BloomFilteringPostingsFormat.java | 13 ++-
.../codecs/bloom/DefaultBloomFilterFactory.java | 2 +-
.../org/apache/lucene/codecs/bloom/FuzzySet.java | 129 +++++++++++----------
.../apache/lucene/codecs/bloom/HashFunction.java | 2 +-
.../apache/lucene/codecs/bloom/MurmurHash2.java | 101 ----------------
.../apache/lucene/codecs/bloom/MurmurHash64.java | 85 ++++++++++++++
8 files changed, 166 insertions(+), 175 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4d7f38d7678..c763b467d18 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -19,7 +19,10 @@ Improvements
Optimizations
---------------------
-(No changes)
+
+* GITHUB#11900: BloomFilteringPostingsFormat now uses multiple hash functions
+ in order to achieve the same false positive probability with less memory.
+ (Jean-François Boeuf)
Bug Fixes
---------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java
index 69f31f19585..910f59a5ab9 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java
@@ -42,9 +42,7 @@ public abstract class BloomFilterFactory {
* @return null or a hopefully more densely packed, smaller bitset
*/
public FuzzySet downsize(FieldInfo fieldInfo, FuzzySet initialSet) {
- // Aim for a bitset size that would have 10% of bits set (so 90% of searches
- // would fail-fast)
- float targetMaxSaturation = 0.1f;
+ float targetMaxSaturation = initialSet.getTargetMaxSaturation();
return initialSet.downsize(targetMaxSaturation);
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
index 52a2e838261..7e5bf64fbbe 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
@@ -53,7 +53,7 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
*
* <p>A choice of {@link BloomFilterFactory} can be passed to tailor Bloom Filter settings on a
* per-field basis. The default configuration is {@link DefaultBloomFilterFactory} which allocates a
- * ~8mb bitset and hashes values using {@link MurmurHash2}. This should be suitable for most
+ * ~8mb bitset and hashes values using {@link MurmurHash64}. This should be suitable for most
* purposes.
*
* <p>The format of the blm file is as follows:
@@ -83,8 +83,8 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
/** Extension of Bloom Filters file */
static final String BLOOM_EXTENSION = "blm";
- BloomFilterFactory bloomFilterFactory = new DefaultBloomFilterFactory();
- private PostingsFormat delegatePostingsFormat;
+ private final BloomFilterFactory bloomFilterFactory;
+ private final PostingsFormat delegatePostingsFormat;
/**
* Creates Bloom filters for a selection of fields created in the index. This is recorded as a set
@@ -120,7 +120,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
// Used only by core Lucene at read-time via Service Provider instantiation -
// do not use at Write-time in application code.
public BloomFilteringPostingsFormat() {
- super(BLOOM_CODEC_NAME);
+ this(null, new DefaultBloomFilterFactory());
}
@Override
@@ -366,6 +366,11 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
public ImpactsEnum impacts(int flags) throws IOException {
return delegate().impacts(flags);
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(filter=" + filter.toString() + ")";
+ }
}
@Override
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java
index 61e60dcbca1..306fdb9b4f9 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java
@@ -31,7 +31,7 @@ public class DefaultBloomFilterFactory extends BloomFilterFactory {
public FuzzySet getSetForField(SegmentWriteState state, FieldInfo info) {
// Assume all of the docs have a unique term (e.g. a primary key) and we hope to maintain a set
// with 10% of bits set
- return FuzzySet.createSetBasedOnQuality(state.segmentInfo.maxDoc(), 0.10f);
+ return FuzzySet.createOptimalSet(state.segmentInfo.maxDoc(), 0.1023f);
}
@Override
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
index 5ccb04203da..f1d2dee65c7 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
@@ -44,21 +44,6 @@ import org.apache.lucene.util.RamUsageEstimator;
*/
public class FuzzySet implements Accountable {
- public static final int VERSION_SPI = 1; // HashFunction used to be loaded through a SPI
- public static final int VERSION_START = VERSION_SPI;
- public static final int VERSION_CURRENT = 2;
-
- public static HashFunction hashFunctionForVersion(int version) {
- if (version < VERSION_START) {
- throw new IllegalArgumentException(
- "Version " + version + " is too old, expected at least " + VERSION_START);
- } else if (version > VERSION_CURRENT) {
- throw new IllegalArgumentException(
- "Version " + version + " is too new, expected at most " + VERSION_CURRENT);
- }
- return MurmurHash2.INSTANCE;
- }
-
/**
* Result from {@link FuzzySet#contains(BytesRef)}: can never return definitively YES (always
* MAYBE), but can sometimes definitely return NO.
@@ -71,6 +56,7 @@ public class FuzzySet implements Accountable {
private HashFunction hashFunction;
private FixedBitSet filter;
private int bloomSize;
+ private final int hashCount;
// The sizes of BitSet used are all numbers that, when expressed in binary form,
// are all ones. This is to enable fast downsizing from one bitset to another
@@ -82,12 +68,9 @@ public class FuzzySet implements Accountable {
static final int[] usableBitSetSizes;
static {
- usableBitSetSizes = new int[30];
- int mask = 1;
- int size = mask;
+ usableBitSetSizes = new int[26];
for (int i = 0; i < usableBitSetSizes.length; i++) {
- size = (size << 1) | mask;
- usableBitSetSizes[i] = size;
+ usableBitSetSizes[i] = (1 << (i + 6)) - 1;
}
}
@@ -131,48 +114,60 @@ public class FuzzySet implements Accountable {
public static FuzzySet createSetBasedOnMaxMemory(int maxNumBytes) {
int setSize = getNearestSetSize(maxNumBytes);
- return new FuzzySet(
- new FixedBitSet(setSize + 1), setSize, hashFunctionForVersion(VERSION_CURRENT));
+ return new FuzzySet(new FixedBitSet(setSize + 1), setSize, 1);
}
public static FuzzySet createSetBasedOnQuality(
- int maxNumUniqueValues, float desiredMaxSaturation) {
+ int maxNumUniqueValues, float desiredMaxSaturation, int version) {
int setSize = getNearestSetSize(maxNumUniqueValues, desiredMaxSaturation);
- return new FuzzySet(
- new FixedBitSet(setSize + 1), setSize, hashFunctionForVersion(VERSION_CURRENT));
+ return new FuzzySet(new FixedBitSet(setSize + 1), setSize, 1);
+ }
+
+ public static FuzzySet createOptimalSet(int maxNumUniqueValues, float targetMaxFpp) {
+ int setSize =
+ (int)
+ Math.ceil(
+ (maxNumUniqueValues * Math.log(targetMaxFpp))
+ / Math.log(1 / Math.pow(2, Math.log(2))));
+ setSize = getNearestSetSize(2 * setSize);
+ int optimalK = (int) Math.round(((double) setSize / maxNumUniqueValues) * Math.log(2));
+ return new FuzzySet(new FixedBitSet(setSize + 1), setSize, optimalK);
}
- private FuzzySet(FixedBitSet filter, int bloomSize, HashFunction hashFunction) {
+ private FuzzySet(FixedBitSet filter, int bloomSize, int hashCount) {
super();
this.filter = filter;
this.bloomSize = bloomSize;
- this.hashFunction = hashFunction;
+ this.hashFunction = MurmurHash64.INSTANCE;
+ this.hashCount = hashCount;
}
/**
* The main method required for a Bloom filter which, given a value determines set membership.
- * Unlike a conventional set, the fuzzy set returns NO or MAYBE rather than true or false.
+ * Unlike a conventional set, the fuzzy set returns NO or MAYBE rather than true or false. Hash
+ * generation follows the same principles as {@link #addValue(BytesRef)}
*
* @return NO or MAYBE
*/
public ContainsResult contains(BytesRef value) {
- int hash = hashFunction.hash(value);
- if (hash < 0) {
- hash = hash * -1;
+ long hash = hashFunction.hash(value);
+ int msb = (int) (hash >>> Integer.SIZE);
+ int lsb = (int) hash;
+ for (int i = 0; i < hashCount; i++) {
+ int bloomPos = (lsb + i * msb);
+ if (!mayContainValue(bloomPos)) {
+ return ContainsResult.NO;
+ }
}
- return mayContainValue(hash);
+ return ContainsResult.MAYBE;
}
/**
* Serializes the data set to file using the following format:
*
* <ul>
- * <li>FuzzySet -->FuzzySetVersion,HashFunctionName,BloomSize,
- * NumBitSetWords,BitSetWord<sup>NumBitSetWords</sup>
- * <li>HashFunctionName --> {@link DataOutput#writeString(String) String} The name of a
- * ServiceProvider registered {@link HashFunction}
- * <li>FuzzySetVersion --> {@link DataOutput#writeInt Uint32} The version number of the
- * {@link FuzzySet} class
+ * <li>FuzzySet -->hashCount,BloomSize, NumBitSetWords,BitSetWord<sup>NumBitSetWords</sup>
+ * <li>hashCount --> {@link DataOutput#writeVInt Uint32} The number of hash functions (k).
* <li>BloomSize --> {@link DataOutput#writeInt Uint32} The modulo value used to project
* hashes into the field's Bitset
* <li>NumBitSetWords --> {@link DataOutput#writeInt Uint32} The number of longs (as returned
@@ -185,7 +180,7 @@ public class FuzzySet implements Accountable {
* @throws IOException If there is a low-level I/O error
*/
public void serialize(DataOutput out) throws IOException {
- out.writeInt(VERSION_CURRENT);
+ out.writeVInt(hashCount);
out.writeInt(bloomSize);
long[] bits = filter.getBits();
out.writeInt(bits.length);
@@ -197,11 +192,7 @@ public class FuzzySet implements Accountable {
}
public static FuzzySet deserialize(DataInput in) throws IOException {
- int version = in.readInt();
- if (version == VERSION_SPI) {
- in.readString();
- }
- final HashFunction hashFunction = hashFunctionForVersion(version);
+ int hashCount = in.readVInt();
int bloomSize = in.readInt();
int numLongs = in.readInt();
long[] longs = new long[numLongs];
@@ -209,36 +200,33 @@ public class FuzzySet implements Accountable {
longs[i] = in.readLong();
}
FixedBitSet bits = new FixedBitSet(longs, bloomSize + 1);
- return new FuzzySet(bits, bloomSize, hashFunction);
+ return new FuzzySet(bits, bloomSize, hashCount);
}
- private ContainsResult mayContainValue(int positiveHash) {
- assert positiveHash >= 0;
+ private boolean mayContainValue(int aHash) {
// Bloom sizes are always base 2 and so can be ANDed for a fast modulo
- int pos = positiveHash & bloomSize;
- if (filter.get(pos)) {
- // This term may be recorded in this index (but could be a collision)
- return ContainsResult.MAYBE;
- }
- // definitely NOT in this segment
- return ContainsResult.NO;
+ int pos = aHash & bloomSize;
+ return filter.get(pos);
}
/**
- * Records a value in the set. The referenced bytes are hashed and then modulo n'd where n is the
- * chosen size of the internal bitset.
+ * Records a value in the set. The referenced bytes are hashed. From the 64-bit generated hash,
+ * two 32-bit hashes are derived from the msb and lsb which can be used to derive more hashes (see
+ * https://www.eecs.harvard.edu/~michaelm/postscripts/rsa2008.pdf). Finally, each generated hash
+ * is modulo n'd where n is the chosen size of the internal bitset.
*
* @param value the key value to be hashed
* @throws IOException If there is a low-level I/O error
*/
public void addValue(BytesRef value) throws IOException {
- int hash = hashFunction.hash(value);
- if (hash < 0) {
- hash = hash * -1;
+ long hash = hashFunction.hash(value);
+ int msb = (int) (hash >>> Integer.SIZE);
+ int lsb = (int) hash;
+ for (int i = 0; i < hashCount; i++) {
+ // Bitmasking using bloomSize is effectively a modulo operation.
+ int bloomPos = (lsb + i * msb) & bloomSize;
+ filter.set(bloomPos);
}
- // Bitmasking using bloomSize is effectively a modulo operation.
- int bloomPos = hash & bloomSize;
- filter.set(bloomPos);
}
/**
@@ -279,7 +267,7 @@ public class FuzzySet implements Accountable {
} else {
return null;
}
- return new FuzzySet(rightSizedBitSet, rightSizedBitSetSize, hashFunction);
+ return new FuzzySet(rightSizedBitSet, rightSizedBitSetSize, hashCount);
}
public int getEstimatedUniqueValues() {
@@ -297,6 +285,10 @@ public class FuzzySet implements Accountable {
return (int) (setSizeAsDouble * logInverseSaturation);
}
+ public float getTargetMaxSaturation() {
+ return 0.5f;
+ }
+
public float getSaturation() {
int numBitsSet = filter.cardinality();
return (float) numBitsSet / (float) bloomSize;
@@ -309,6 +301,15 @@ public class FuzzySet implements Accountable {
@Override
public String toString() {
- return getClass().getSimpleName() + "(hash=" + hashFunction + ")";
+ return getClass().getSimpleName()
+ + "(hash="
+ + hashFunction
+ + ", k="
+ + hashCount
+ + ", bits="
+ + filter.cardinality()
+ + "/"
+ + filter.length()
+ + ")";
}
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java
index 5b22d121d2e..eac514a7bb8 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java
@@ -33,5 +33,5 @@ public abstract class HashFunction {
* @param bytes the data to be hashed
* @return the hash of the bytes referenced by bytes.offset and length bytes.length
*/
- public abstract int hash(BytesRef bytes);
+ public abstract long hash(BytesRef bytes);
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java
deleted file mode 100644
index c3f5853f409..00000000000
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.bloom;
-
-import org.apache.lucene.util.BytesRef;
-
-/**
- * This is a very fast, non-cryptographic hash suitable for general hash-based lookup. See
- * http://murmurhash.googlepages.com/ for more details.
- *
- * <p>The C version of MurmurHash 2.0 found at that site was ported to Java by Andrzej Bialecki (ab
- * at getopt org).
- *
- * <p>The code from getopt.org was adapted by Mark Harwood in the form here as one of a pluggable
- * choice of hashing functions as the core function had to be adapted to work with BytesRefs with
- * offsets and lengths rather than raw byte arrays.
- *
- * @lucene.experimental
- */
-public final class MurmurHash2 extends HashFunction {
-
- public static final MurmurHash2 INSTANCE = new MurmurHash2();
-
- private MurmurHash2() {}
-
- public static int hash(byte[] data, int seed, int offset, int len) {
- int m = 0x5bd1e995;
- int r = 24;
- int h = seed ^ len;
- int len_4 = len >> 2;
- for (int i = 0; i < len_4; i++) {
- int i_4 = offset + (i << 2);
- int k = data[i_4 + 3];
- k = k << 8;
- k = k | (data[i_4 + 2] & 0xff);
- k = k << 8;
- k = k | (data[i_4 + 1] & 0xff);
- k = k << 8;
- k = k | (data[i_4 + 0] & 0xff);
- k *= m;
- k ^= k >>> r;
- k *= m;
- h *= m;
- h ^= k;
- }
- int len_m = len_4 << 2;
- int left = len - len_m;
- if (left != 0) {
- if (left >= 3) {
- h ^= data[offset + len - 3] << 16;
- }
- if (left >= 2) {
- h ^= data[offset + len - 2] << 8;
- }
- if (left >= 1) {
- h ^= data[offset + len - 1];
- }
- h *= m;
- }
- h ^= h >>> 13;
- h *= m;
- h ^= h >>> 15;
- return h;
- }
-
- /**
- * Generates 32 bit hash from byte array with default seed value.
- *
- * @param data byte array to hash
- * @param offset the start position in the array to hash
- * @param len length of the array elements to hash
- * @return 32 bit hash of the given array
- */
- public static final int hash32(final byte[] data, int offset, int len) {
- return MurmurHash2.hash(data, 0x9747b28c, offset, len);
- }
-
- @Override
- public final int hash(BytesRef br) {
- return hash32(br.bytes, br.offset, br.length);
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName();
- }
-}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.java
new file mode 100644
index 00000000000..1d189773143
--- /dev/null
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash64.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.bloom;
+
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * This is a very fast, non-cryptographic hash suitable for general hash-based lookup. See
+ * http://murmurhash.googlepages.com/ for more details.
+ *
+ * <p>The code from Apache Commons was adapted in the form here to work with BytesRefs with offsets
+ * and lengths rather than raw byte arrays.
+ */
+public class MurmurHash64 extends HashFunction {
+ private static final long M64 = 0xc6a4a7935bd1e995L;
+ private static final int R64 = 47;
+ public static final HashFunction INSTANCE = new MurmurHash64();
+
+ /**
+ * Generates a 64-bit hash from byte array of the given length and seed.
+ *
+ * @param data The input byte array
+ * @param seed The initial seed value
+ * @param length The length of the array
+ * @return The 64-bit hash of the given array
+ */
+ public static long hash64(byte[] data, int seed, int offset, int length) {
+ long h = (seed & 0xffffffffL) ^ (length * M64);
+
+ final int nblocks = length >> 3;
+
+ // body
+ for (int i = 0; i < nblocks; i++) {
+
+ long k = (long) BitUtil.VH_LE_LONG.get(data, offset);
+ k *= M64;
+ k ^= k >>> R64;
+ k *= M64;
+
+ h ^= k;
+ h *= M64;
+
+ offset += Long.BYTES;
+ }
+
+ int remaining = length & 0x07;
+ if (0 < remaining) {
+ for (int i = 0; i < remaining; i++) {
+ h ^= ((long) data[offset + i] & 0xff) << (Byte.SIZE * i);
+ }
+ h *= M64;
+ }
+
+ h ^= h >>> R64;
+ h *= M64;
+ h ^= h >>> R64;
+
+ return h;
+ }
+
+ @Override
+ public final long hash(BytesRef br) {
+ return hash64(br.bytes, 0xe17a1465, br.offset, br.length);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+}
[lucene] 01/02: GITHUB-11868: Add FilterIndexInput and FilterIndexOutput wrapper classes (#11958)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 6df5f0e170973d2e5b154787056c20810a7ec53d
Author: Marc D'Mello <ma...@gmail.com>
AuthorDate: Wed Feb 1 05:11:12 2023 -0800
GITHUB-11868: Add FilterIndexInput and FilterIndexOutput wrapper classes (#11958)
Co-authored-by: Marc D'Mello <dm...@amazon.com>
---
lucene/CHANGES.txt | 4 +-
.../store/EndiannessReverserIndexInput.java | 90 +-------------------
.../store/EndiannessReverserIndexOutput.java | 67 +--------------
.../store/EndiannessReverserUtil.java | 4 +-
.../lucene/store/ByteBuffersIndexOutput.java | 4 +
.../org/apache/lucene/store/FilterIndexInput.java | 90 ++++++++++++++++++++
.../org/apache/lucene/store/FilterIndexOutput.java | 84 +++++++++++++++++++
.../lucene/store/RateLimitedIndexOutput.java | 33 ++------
.../apache/lucene/index/TestFilterIndexInput.java | 95 ++++++++++++++++++++++
.../org/apache/lucene/index/TestIndexInput.java | 22 +++--
.../apache/lucene/index/TestLazyProxSkipping.java | 41 ++--------
.../lucene/index/TestMultiLevelSkipList.java | 35 ++------
.../apache/lucene/store/TestFilterIndexOutput.java | 67 +++++++++++++++
.../org/apache/lucene/facet/SlowDirectory.java | 74 +++--------------
.../lucene/misc/store/ByteTrackingIndexOutput.java | 61 ++------------
.../tests/index/BaseIndexFileFormatTestCase.java | 27 +-----
.../lucene/tests/store/CorruptingIndexOutput.java | 22 +----
.../lucene/tests/store/MockIndexInputWrapper.java | 51 ++++++------
.../lucene/tests/store/MockIndexOutputWrapper.java | 43 ++++------
.../lucene/tests/util/ThrottledIndexOutput.java | 48 +++--------
20 files changed, 466 insertions(+), 496 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 950ddd120ee..4d7f38d7678 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -27,7 +27,9 @@ Bug Fixes
Other
---------------------
-(No changes)
+
+* GITHUB#11868: Add a FilterIndexInput and FilterIndexOutput class to more easily and safely create delegate
+ IndexInput and IndexOutput classes (Marc D'Mello)
======================== Lucene 9.5.0 =======================
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java
index 4a1c0566ffc..b0a2b3ad57e 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexInput.java
@@ -17,34 +17,15 @@
package org.apache.lucene.backward_codecs.store;
import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
/** A {@link IndexInput} wrapper that changes the endianness of the provided index input. */
-final class EndiannessReverserIndexInput extends IndexInput {
-
- final IndexInput in;
+final class EndiannessReverserIndexInput extends FilterIndexInput {
EndiannessReverserIndexInput(IndexInput in) {
- super("Endianness reverser Index Input wrapper");
- this.in = in;
- }
-
- @Override
- public byte readByte() throws IOException {
- return in.readByte();
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- in.readBytes(b, offset, len);
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
- in.readBytes(b, offset, len, useBuffer);
+ super("Endianness reverser Index Input wrapper", in);
}
@Override
@@ -57,16 +38,6 @@ final class EndiannessReverserIndexInput extends IndexInput {
return Integer.reverseBytes(in.readInt());
}
- @Override
- public int readVInt() throws IOException {
- return in.readVInt();
- }
-
- @Override
- public int readZInt() throws IOException {
- return in.readZInt();
- }
-
@Override
public long readLong() throws IOException {
return Long.reverseBytes(in.readLong());
@@ -97,21 +68,6 @@ final class EndiannessReverserIndexInput extends IndexInput {
}
}
- @Override
- public long readVLong() throws IOException {
- return in.readVLong();
- }
-
- @Override
- public long readZLong() throws IOException {
- return in.readZLong();
- }
-
- @Override
- public String readString() throws IOException {
- return in.readString();
- }
-
@Override
public IndexInput clone() {
return new EndiannessReverserIndexInput(in.clone());
@@ -122,46 +78,6 @@ final class EndiannessReverserIndexInput extends IndexInput {
return new EndiannessReverserIndexInput(in.slice(sliceDescription, offset, length));
}
- @Override
- public Map<String, String> readMapOfStrings() throws IOException {
- return in.readMapOfStrings();
- }
-
- @Override
- public Set<String> readSetOfStrings() throws IOException {
- return in.readSetOfStrings();
- }
-
- @Override
- public void skipBytes(final long numBytes) throws IOException {
- in.skipBytes(numBytes);
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- }
-
- @Override
- public long getFilePointer() {
- return in.getFilePointer();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- in.seek(pos);
- }
-
- @Override
- public long length() {
- return in.length();
- }
-
- @Override
- public String toString() {
- return in.toString();
- }
-
@Override
public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
return new EndiannessReverserRandomAccessInput(in.randomAccessSlice(offset, length));
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexOutput.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexOutput.java
index 7da9909da70..f1d94808bc0 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexOutput.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserIndexOutput.java
@@ -17,40 +17,14 @@
package org.apache.lucene.backward_codecs.store;
import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.FilterIndexOutput;
import org.apache.lucene.store.IndexOutput;
/** A {@link IndexOutput} wrapper that changes the endianness of the provided index output. */
-final class EndiannessReverserIndexOutput extends IndexOutput {
-
- final IndexOutput out;
+final class EndiannessReverserIndexOutput extends FilterIndexOutput {
EndiannessReverserIndexOutput(IndexOutput out) {
- super("Endianness reverser Index Output wrapper", out.getName());
- this.out = out;
- }
-
- @Override
- public String getName() {
- return out.getName();
- }
-
- /** Closes this stream to further operations. */
- @Override
- public void close() throws IOException {
- out.close();
- }
-
- @Override
- public long getFilePointer() {
- return out.getFilePointer();
- }
-
- @Override
- public long getChecksum() throws IOException {
- return out.getChecksum();
+ super("Endianness reverser Index Output wrapper", out.getName(), out);
}
@Override
@@ -58,21 +32,6 @@ final class EndiannessReverserIndexOutput extends IndexOutput {
return out.getName();
}
- @Override
- public void writeByte(byte b) throws IOException {
- out.writeByte(b);
- }
-
- @Override
- public void writeBytes(byte[] b, int length) throws IOException {
- writeBytes(b, 0, length);
- }
-
- @Override
- public void writeBytes(byte[] b, int offset, int length) throws IOException {
- out.writeBytes(b, offset, length);
- }
-
@Override
public void writeInt(int i) throws IOException {
out.writeInt(Integer.reverseBytes(i));
@@ -87,24 +46,4 @@ final class EndiannessReverserIndexOutput extends IndexOutput {
public void writeLong(long i) throws IOException {
out.writeLong(Long.reverseBytes(i));
}
-
- @Override
- public void writeString(String s) throws IOException {
- out.writeString(s);
- }
-
- @Override
- public void copyBytes(DataInput input, long numBytes) throws IOException {
- out.copyBytes(input, numBytes);
- }
-
- @Override
- public void writeMapOfStrings(Map<String, String> map) throws IOException {
- out.writeMapOfStrings(map);
- }
-
- @Override
- public void writeSetOfStrings(Set<String> set) throws IOException {
- out.writeSetOfStrings(set);
- }
}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserUtil.java
index 44e3c217ccd..8a1dbbec9d9 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/store/EndiannessReverserUtil.java
@@ -66,7 +66,7 @@ public final class EndiannessReverserUtil {
return ((EndiannessReverserDataOutput) dataOutput).out;
}
if (dataOutput instanceof EndiannessReverserIndexOutput) {
- return ((EndiannessReverserIndexOutput) dataOutput).out;
+ return ((EndiannessReverserIndexOutput) dataOutput).getDelegate();
}
return new EndiannessReverserDataOutput(dataOutput);
}
@@ -77,7 +77,7 @@ public final class EndiannessReverserUtil {
return ((EndiannessReverserDataInput) dataInput).in;
}
if (dataInput instanceof EndiannessReverserIndexInput) {
- return ((EndiannessReverserIndexInput) dataInput).in;
+ return ((EndiannessReverserIndexInput) dataInput).getDelegate();
}
return new EndiannessReverserDataInput(dataInput);
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java
index 0a7ed683edb..fdbe8e1dbea 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexOutput.java
@@ -159,4 +159,8 @@ public final class ByteBuffersIndexOutput extends IndexOutput {
throw new AlreadyClosedException("Already closed.");
}
}
+
+ public byte[] toArrayCopy() {
+ return delegate.toArrayCopy();
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/FilterIndexInput.java
new file mode 100644
index 00000000000..5b4a5c506ee
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/store/FilterIndexInput.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+import java.io.IOException;
+
+/**
+ * IndexInput implementation that delegates calls to another directory. This class can be used to
+ * add limitations on top of an existing {@link IndexInput} implementation or to add additional
+ * sanity checks for tests. However, if you plan to write your own {@link IndexInput}
+ * implementation, you should consider extending directly {@link IndexInput} or {@link DataInput}
+ * rather than try to reuse functionality of existing {@link IndexInput}s by extending this class.
+ *
+ * @lucene.internal
+ */
+public class FilterIndexInput extends IndexInput {
+
+ /**
+ * Unwraps all FilterIndexInputs until the first non-FilterIndexInput IndexInput instance and
+ * returns it
+ */
+ public static IndexInput unwrap(IndexInput in) {
+ while (in instanceof FilterIndexInput) {
+ in = ((FilterIndexInput) in).in;
+ }
+ return in;
+ }
+
+ protected final IndexInput in;
+
+ /** Creates a FilterIndexInput with a resource description and wrapped delegate IndexInput */
+ public FilterIndexInput(String resourceDescription, IndexInput in) {
+ super(resourceDescription);
+ this.in = in;
+ }
+
+ /** Gets the delegate that was passed in on creation */
+ public IndexInput getDelegate() {
+ return in;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public long getFilePointer() {
+ return in.getFilePointer();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ in.seek(pos);
+ }
+
+ @Override
+ public long length() {
+ return in.length();
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ return in.slice(sliceDescription, offset, length);
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ return in.readByte();
+ }
+
+ @Override
+ public void readBytes(byte[] b, int offset, int len) throws IOException {
+ in.readBytes(b, offset, len);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/store/FilterIndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/FilterIndexOutput.java
new file mode 100644
index 00000000000..1faec626d08
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/store/FilterIndexOutput.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.store;
+
+import java.io.IOException;
+
+/**
+ * IndexOutput implementation that delegates calls to another directory. This class can be used to
+ * add limitations on top of an existing {@link IndexOutput} implementation such as {@link
+ * ByteBuffersIndexOutput} or to add additional sanity checks for tests. However, if you plan to
+ * write your own {@link IndexOutput} implementation, you should consider extending directly {@link
+ * IndexOutput} or {@link DataOutput} rather than try to reuse functionality of existing {@link
+ * IndexOutput}s by extending this class.
+ *
+ * @lucene.internal
+ */
+public class FilterIndexOutput extends IndexOutput {
+
+ /**
+ * Unwraps all FilterIndexOutputs until the first non-FilterIndexOutput IndexOutput instance and
+ * returns it
+ */
+ public static IndexOutput unwrap(IndexOutput out) {
+ while (out instanceof FilterIndexOutput) {
+ out = ((FilterIndexOutput) out).out;
+ }
+ return out;
+ }
+
+ protected final IndexOutput out;
+
+ /**
+ * Creates a FilterIndexOutput with a resource description, name, and wrapped delegate IndexOutput
+ */
+ protected FilterIndexOutput(String resourceDescription, String name, IndexOutput out) {
+ super(resourceDescription, name);
+ this.out = out;
+ }
+
+ /** Gets the delegate that was passed in on creation */
+ public final IndexOutput getDelegate() {
+ return out;
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.close();
+ }
+
+ @Override
+ public long getFilePointer() {
+ return out.getFilePointer();
+ }
+
+ @Override
+ public long getChecksum() throws IOException {
+ return out.getChecksum();
+ }
+
+ @Override
+ public void writeByte(byte b) throws IOException {
+ out.writeByte(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) throws IOException {
+ out.writeBytes(b, offset, length);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
index d5d5709bca4..bfac505b8b0 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RateLimitedIndexOutput.java
@@ -23,9 +23,8 @@ import java.io.IOException;
*
* @lucene.internal
*/
-public final class RateLimitedIndexOutput extends IndexOutput {
+public final class RateLimitedIndexOutput extends FilterIndexOutput {
- private final IndexOutput delegate;
private final RateLimiter rateLimiter;
/** How many bytes we've written since we last called rateLimiter.pause. */
@@ -37,33 +36,17 @@ public final class RateLimitedIndexOutput extends IndexOutput {
*/
private long currentMinPauseCheckBytes;
- public RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput delegate) {
- super("RateLimitedIndexOutput(" + delegate + ")", delegate.getName());
- this.delegate = delegate;
+ public RateLimitedIndexOutput(final RateLimiter rateLimiter, final IndexOutput out) {
+ super("RateLimitedIndexOutput(" + out + ")", out.getName(), out);
this.rateLimiter = rateLimiter;
this.currentMinPauseCheckBytes = rateLimiter.getMinPauseCheckBytes();
}
- @Override
- public void close() throws IOException {
- delegate.close();
- }
-
- @Override
- public long getFilePointer() {
- return delegate.getFilePointer();
- }
-
- @Override
- public long getChecksum() throws IOException {
- return delegate.getChecksum();
- }
-
@Override
public void writeByte(byte b) throws IOException {
bytesSinceLastPause++;
checkRate();
- delegate.writeByte(b);
+ out.writeByte(b);
}
@Override
@@ -74,28 +57,28 @@ public final class RateLimitedIndexOutput extends IndexOutput {
// This can cause instant write rate to breach rate limit if there have
// been no writes for enough time to keep the average write rate within limit.
// See https://issues.apache.org/jira/browse/LUCENE-10448
- delegate.writeBytes(b, offset, length);
+ out.writeBytes(b, offset, length);
}
@Override
public void writeInt(int i) throws IOException {
bytesSinceLastPause += Integer.BYTES;
checkRate();
- delegate.writeInt(i);
+ out.writeInt(i);
}
@Override
public void writeShort(short i) throws IOException {
bytesSinceLastPause += Short.BYTES;
checkRate();
- delegate.writeShort(i);
+ out.writeShort(i);
}
@Override
public void writeLong(long i) throws IOException {
bytesSinceLastPause += Long.BYTES;
checkRate();
- delegate.writeLong(i);
+ out.writeLong(i);
}
private void checkRate() throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterIndexInput.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterIndexInput.java
new file mode 100644
index 00000000000..d43b580f601
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterIndexInput.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Random;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.junit.Test;
+
+public class TestFilterIndexInput extends TestIndexInput {
+
+ @Override
+ public IndexInput getIndexInput(long len) {
+ return new FilterIndexInput("wrapped foo", new InterceptingIndexInput("foo", len));
+ }
+
+ public void testRawFilterIndexInputRead() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ Random random = random();
+ final Directory dir = newDirectory();
+ IndexOutput os = dir.createOutput("foo", newIOContext(random));
+ os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length);
+ os.close();
+ IndexInput is =
+ new FilterIndexInput("wrapped foo", dir.openInput("foo", newIOContext(random)));
+ checkReads(is, IOException.class);
+ checkSeeksAndSkips(is, random);
+ is.close();
+
+ os = dir.createOutput("bar", newIOContext(random));
+ os.writeBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.length);
+ os.close();
+ is = new FilterIndexInput("wrapped bar", dir.openInput("bar", newIOContext(random)));
+ checkRandomReads(is);
+ checkSeeksAndSkips(is, random);
+ is.close();
+ dir.close();
+ }
+ }
+
+ @Test
+ public void testOverrides() {
+ for (Method m : FilterIndexInput.class.getMethods()) {
+ if (m.getName().contains("clone")) {
+ // special case
+ continue;
+ }
+ if (m.getDeclaringClass() == FilterIndexInput.class) {
+ // verify that only abstract methods are overridden
+ Method indexInputMethod;
+ try {
+ indexInputMethod = IndexInput.class.getMethod(m.getName(), m.getParameterTypes());
+ assertTrue(
+ "Non-abstract method " + m.getName() + " is overridden",
+ Modifier.isAbstract(indexInputMethod.getModifiers()));
+ } catch (Exception e) {
+ assertTrue(e instanceof NoSuchMethodException);
+ }
+ }
+ }
+ }
+
+ public void testUnwrap() throws IOException {
+ Directory dir = FSDirectory.open(createTempDir());
+ IndexOutput ignored = dir.createOutput("test", IOContext.DEFAULT);
+ IndexInput indexInput = dir.openInput("test", IOContext.DEFAULT);
+ FilterIndexInput filterIndexInput = new FilterIndexInput("wrapper of test", indexInput);
+ assertEquals(indexInput, filterIndexInput.getDelegate());
+ assertEquals(indexInput, FilterIndexInput.unwrap(filterIndexInput));
+ ignored.close();
+ filterIndexInput.close();
+ dir.close();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
index 3d52aa36fee..8eba45549ad 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
@@ -211,7 +211,7 @@ public class TestIndexInput extends LuceneTestCase {
RANDOM_TEST_BYTES = null;
}
- private void checkReads(DataInput is, Class<? extends Exception> expectedEx) throws IOException {
+ void checkReads(DataInput is, Class<? extends Exception> expectedEx) throws IOException {
assertEquals(128, is.readVInt());
assertEquals(16383, is.readVInt());
assertEquals(16384, is.readVInt());
@@ -254,7 +254,7 @@ public class TestIndexInput extends LuceneTestCase {
assertEquals(1L, is.readVLong()); // guard value
}
- private void checkRandomReads(DataInput is) throws IOException {
+ void checkRandomReads(DataInput is) throws IOException {
for (int i = 0; i < COUNT; i++) {
assertEquals(INTS[i], is.readVInt());
assertEquals(INTS[i], is.readInt());
@@ -263,7 +263,7 @@ public class TestIndexInput extends LuceneTestCase {
}
}
- private void checkSeeksAndSkips(IndexInput is, Random random) throws IOException {
+ void checkSeeksAndSkips(IndexInput is, Random random) throws IOException {
long len = is.length();
int iterations = LuceneTestCase.TEST_NIGHTLY ? 1_000 : 10;
@@ -332,21 +332,25 @@ public class TestIndexInput extends LuceneTestCase {
public void testNoReadOnSkipBytes() throws IOException {
long len = LuceneTestCase.TEST_NIGHTLY ? Long.MAX_VALUE : 1_000_000;
long maxSeekPos = len - 1;
- InterceptingIndexInput is = new InterceptingIndexInput("foo", len);
+ IndexInput is = getIndexInput(len);
- while (is.pos < maxSeekPos) {
- long seekPos = TestUtil.nextLong(random(), is.pos, maxSeekPos);
- long skipDelta = seekPos - is.pos;
+ while (is.getFilePointer() < maxSeekPos) {
+ long seekPos = TestUtil.nextLong(random(), is.getFilePointer(), maxSeekPos);
+ long skipDelta = seekPos - is.getFilePointer();
is.skipBytes(skipDelta);
- assertEquals(seekPos, is.pos);
+ assertEquals(seekPos, is.getFilePointer());
}
}
+ public IndexInput getIndexInput(long len) {
+ return new InterceptingIndexInput("foo", len);
+ }
+
/**
* Mock IndexInput that just tracks a position (which responds to seek/skip) and throws if
* #readByte or #readBytes are called, ensuring seek/skip doesn't invoke reads.
*/
- private static final class InterceptingIndexInput extends IndexInput {
+ protected static final class InterceptingIndexInput extends IndexInput {
long pos = 0;
final long len;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
index d6c84895152..310d8524be9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -174,53 +175,21 @@ public class TestLazyProxSkipping extends LuceneTestCase {
// Simply extends IndexInput in a way that we are able to count the number
// of invocations of seek()
- class SeeksCountingStream extends IndexInput {
- private IndexInput input;
+ class SeeksCountingStream extends FilterIndexInput {
SeeksCountingStream(IndexInput input) {
- super("SeekCountingStream(" + input + ")");
- this.input = input;
- }
-
- @Override
- public byte readByte() throws IOException {
- return this.input.readByte();
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- this.input.readBytes(b, offset, len);
- }
-
- @Override
- public void close() throws IOException {
- this.input.close();
- }
-
- @Override
- public long getFilePointer() {
- return this.input.getFilePointer();
+ super("SeekCountingStream(" + input + ")", input);
}
@Override
public void seek(long pos) throws IOException {
TestLazyProxSkipping.this.seeksCounter++;
- this.input.seek(pos);
- }
-
- @Override
- public long length() {
- return this.input.length();
+ in.seek(pos);
}
@Override
public SeeksCountingStream clone() {
- return new SeeksCountingStream(this.input.clone());
- }
-
- @Override
- public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
- return new SeeksCountingStream(this.input.slice(sliceDescription, offset, length));
+ return new SeeksCountingStream(in.clone());
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
index a0d14f50705..0b0e6c8759c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockTokenizer;
@@ -150,54 +151,32 @@ public class TestMultiLevelSkipList extends LuceneTestCase {
// Simply extends IndexInput in a way that we are able to count the number
// of bytes read
- class CountingStream extends IndexInput {
- private IndexInput input;
+ class CountingStream extends FilterIndexInput {
CountingStream(IndexInput input) {
- super("CountingStream(" + input + ")");
- this.input = input;
+ super("CountingStream(" + input + ")", input);
}
@Override
public byte readByte() throws IOException {
TestMultiLevelSkipList.this.counter++;
- return this.input.readByte();
+ return in.readByte();
}
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
TestMultiLevelSkipList.this.counter += len;
- this.input.readBytes(b, offset, len);
- }
-
- @Override
- public void close() throws IOException {
- this.input.close();
- }
-
- @Override
- public long getFilePointer() {
- return this.input.getFilePointer();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- this.input.seek(pos);
- }
-
- @Override
- public long length() {
- return this.input.length();
+ in.readBytes(b, offset, len);
}
@Override
public CountingStream clone() {
- return new CountingStream(this.input.clone());
+ return new CountingStream(in.clone());
}
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
- return new CountingStream(this.input.slice(sliceDescription, offset, length));
+ return new CountingStream(in.slice(sliceDescription, offset, length));
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFilterIndexOutput.java b/lucene/core/src/test/org/apache/lucene/store/TestFilterIndexOutput.java
new file mode 100644
index 00000000000..2abdf754d59
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/store/TestFilterIndexOutput.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import org.junit.Test;
+
+public class TestFilterIndexOutput extends BaseDataOutputTestCase<FilterIndexOutput> {
+
+ @Override
+ protected FilterIndexOutput newInstance() {
+ return new FilterIndexOutput(
+ "test",
+ "test",
+ new ByteBuffersIndexOutput(ByteBuffersDataOutput.newResettableInstance(), "test", "test"));
+ }
+
+ @Override
+ protected byte[] toBytes(FilterIndexOutput instance) {
+ return ((ByteBuffersIndexOutput) instance.out).toArrayCopy();
+ }
+
+ @Test
+ public void testOverrides() {
+ for (Method m : FilterIndexOutput.class.getMethods()) {
+ if (m.getDeclaringClass() == FilterIndexOutput.class) {
+ // verify that only abstract methods are overridden
+ Method indexOutputMethod;
+ try {
+ indexOutputMethod = IndexOutput.class.getMethod(m.getName(), m.getParameterTypes());
+ assertTrue(
+ "Non-abstract method " + m.getName() + " is overridden",
+ Modifier.isAbstract(indexOutputMethod.getModifiers()));
+ } catch (Exception e) {
+ assertTrue(e instanceof NoSuchMethodException);
+ }
+ }
+ }
+ }
+
+ public void testUnwrap() throws IOException {
+ Directory dir = FSDirectory.open(createTempDir());
+ IndexOutput output = dir.createOutput("test", IOContext.DEFAULT);
+ FilterIndexOutput filterIndexOutput =
+ new FilterIndexOutput("wrapper of test", "FilterDirectory{test}", output);
+ assertEquals(output, filterIndexOutput.getDelegate());
+ assertEquals(output, FilterIndexOutput.unwrap(filterIndexOutput));
+ filterIndexOutput.close();
+ dir.close();
+ }
+}
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/SlowDirectory.java b/lucene/facet/src/test/org/apache/lucene/facet/SlowDirectory.java
index a809ea1e711..b0c17742c2f 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/SlowDirectory.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/SlowDirectory.java
@@ -20,6 +20,8 @@ import java.io.IOException;
import java.util.Random;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.FilterIndexOutput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -82,15 +84,13 @@ public class SlowDirectory extends FilterDirectory {
}
/** Delegate class to wrap an IndexInput and delay reading bytes by some specified time. */
- private class SlowIndexInput extends IndexInput {
- private IndexInput ii;
+ private class SlowIndexInput extends FilterIndexInput {
private int numRead = 0;
private Random rand;
public SlowIndexInput(IndexInput ii) {
- super("SlowIndexInput(" + ii + ")");
+ super("SlowIndexInput(" + ii + ")", ii);
this.rand = forkRandom();
- this.ii = ii;
}
@Override
@@ -100,7 +100,7 @@ public class SlowDirectory extends FilterDirectory {
numRead = 0;
}
++numRead;
- return ii.readByte();
+ return in.readByte();
}
@Override
@@ -110,61 +110,28 @@ public class SlowDirectory extends FilterDirectory {
numRead = 0;
}
numRead += len;
- ii.readBytes(b, offset, len);
- }
-
- // TODO: is it intentional that clone doesnt wrap?
- @Override
- public IndexInput clone() {
- return ii.clone();
- }
-
- @Override
- public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
- return ii.slice(sliceDescription, offset, length);
- }
-
- @Override
- public void close() throws IOException {
- ii.close();
+ in.readBytes(b, offset, len);
}
@Override
public boolean equals(Object o) {
- return ii.equals(o);
- }
-
- @Override
- public long getFilePointer() {
- return ii.getFilePointer();
+ return in.equals(o);
}
@Override
public int hashCode() {
- return ii.hashCode();
- }
-
- @Override
- public long length() {
- return ii.length();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- ii.seek(pos);
+ return in.hashCode();
}
}
/** Delegate class to wrap an IndexOutput and delay writing bytes by some specified time. */
- private class SlowIndexOutput extends IndexOutput {
+ private class SlowIndexOutput extends FilterIndexOutput {
- private IndexOutput io;
private int numWrote;
private final Random rand;
- public SlowIndexOutput(IndexOutput io) {
- super("SlowIndexOutput(" + io + ")", io.getName());
- this.io = io;
+ public SlowIndexOutput(IndexOutput out) {
+ super("SlowIndexOutput(" + out + ")", out.getName(), out);
this.rand = forkRandom();
}
@@ -175,7 +142,7 @@ public class SlowDirectory extends FilterDirectory {
numWrote = 0;
}
++numWrote;
- io.writeByte(b);
+ out.writeByte(b);
}
@Override
@@ -185,22 +152,7 @@ public class SlowDirectory extends FilterDirectory {
numWrote = 0;
}
numWrote += length;
- io.writeBytes(b, offset, length);
- }
-
- @Override
- public void close() throws IOException {
- io.close();
- }
-
- @Override
- public long getFilePointer() {
- return io.getFilePointer();
- }
-
- @Override
- public long getChecksum() throws IOException {
- return io.getChecksum();
+ out.writeBytes(b, offset, length);
}
}
}
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/ByteTrackingIndexOutput.java b/lucene/misc/src/java/org/apache/lucene/misc/store/ByteTrackingIndexOutput.java
index 4adb1938b4a..f55c555b368 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/store/ByteTrackingIndexOutput.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/store/ByteTrackingIndexOutput.java
@@ -18,74 +18,31 @@ package org.apache.lucene.misc.store;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.store.FilterIndexOutput;
import org.apache.lucene.store.IndexOutput;
/** An {@link IndexOutput} that wraps another instance and tracks the number of bytes written */
-public class ByteTrackingIndexOutput extends IndexOutput {
+public class ByteTrackingIndexOutput extends FilterIndexOutput {
- private final IndexOutput output;
private final AtomicLong byteTracker;
private boolean closed = false;
- protected ByteTrackingIndexOutput(IndexOutput output, AtomicLong byteTracker) {
+ protected ByteTrackingIndexOutput(IndexOutput out, AtomicLong byteTracker) {
super(
- "Byte tracking wrapper for: " + output.getName(),
- "ByteTrackingIndexOutput{" + output.getName() + "}");
- this.output = output;
+ "Byte tracking wrapper for: " + out.getName(),
+ "ByteTrackingIndexOutput{" + out.getName() + "}",
+ out);
this.byteTracker = byteTracker;
}
- @Override
- public void writeByte(byte b) throws IOException {
- output.writeByte(b);
- }
-
- @Override
- public void writeBytes(byte[] b, int offset, int length) throws IOException {
- output.writeBytes(b, offset, length);
- }
-
- @Override
- public void writeShort(short i) throws IOException {
- output.writeShort(i);
- }
-
- @Override
- public void writeInt(int i) throws IOException {
- output.writeInt(i);
- }
-
- @Override
- public void writeLong(long i) throws IOException {
- output.writeLong(i);
- }
-
@Override
public void close() throws IOException {
if (closed) {
- output.close();
+ out.close();
return;
}
- byteTracker.addAndGet(output.getFilePointer());
+ byteTracker.addAndGet(out.getFilePointer());
closed = true;
- output.close();
- }
-
- @Override
- public long getFilePointer() {
- return output.getFilePointer();
- }
-
- @Override
- public long getChecksum() throws IOException {
- return output.getChecksum();
- }
-
- public String getWrappedName() {
- return output.getName();
- }
-
- public String getWrappedToString() {
- return output.toString();
+ out.close();
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
index 7b39777e930..795f659c49b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java
@@ -78,6 +78,7 @@ import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -780,14 +781,12 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
}
}
- private static class ReadBytesIndexInputWrapper extends IndexInput {
+ private static class ReadBytesIndexInputWrapper extends FilterIndexInput {
- private final IndexInput in;
private final IntConsumer readByte;
ReadBytesIndexInputWrapper(IndexInput in, IntConsumer readByte) {
- super(in.toString());
- this.in = in;
+ super(in.toString(), in);
this.readByte = readByte;
}
@@ -796,26 +795,6 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
return new ReadBytesIndexInputWrapper(in.clone(), readByte);
}
- @Override
- public void close() throws IOException {
- in.close();
- }
-
- @Override
- public long getFilePointer() {
- return in.getFilePointer();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- in.seek(pos);
- }
-
- @Override
- public long length() {
- return in.length();
- }
-
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
IndexInput slice = in.slice(sliceDescription, offset, length);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/CorruptingIndexOutput.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/CorruptingIndexOutput.java
index c0d65b31001..c3c3ebf4313 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/CorruptingIndexOutput.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/CorruptingIndexOutput.java
@@ -19,22 +19,21 @@ package org.apache.lucene.tests.store;
import java.io.IOException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterIndexOutput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
/** Corrupts on bit of a file after close */
-public class CorruptingIndexOutput extends IndexOutput {
- protected final IndexOutput out;
+public class CorruptingIndexOutput extends FilterIndexOutput {
final Directory dir;
final long byteToCorrupt;
private boolean closed;
public CorruptingIndexOutput(Directory dir, long byteToCorrupt, IndexOutput out) {
- super("CorruptingIndexOutput(" + out + ")", out.getName());
+ super("CorruptingIndexOutput(" + out + ")", out.getName(), out);
this.dir = dir;
this.byteToCorrupt = byteToCorrupt;
- this.out = out;
}
@Override
@@ -83,26 +82,11 @@ public class CorruptingIndexOutput extends IndexOutput {
dir.deleteFile(newTempName);
}
- @Override
- public long getFilePointer() {
- return out.getFilePointer();
- }
-
@Override
public long getChecksum() throws IOException {
return out.getChecksum() ^ 1;
}
- @Override
- public String toString() {
- return "CorruptingIndexOutput(" + out + ")";
- }
-
- @Override
- public void writeByte(byte b) throws IOException {
- out.writeByte(b);
- }
-
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
for (int i = 0; i < length; i++) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
index d1b034a1237..39c41d46825 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
@@ -20,15 +20,15 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
+import org.apache.lucene.store.FilterIndexInput;
import org.apache.lucene.store.IndexInput;
/**
* Used by MockDirectoryWrapper to create an input stream that keeps track of when it's been closed.
*/
-public class MockIndexInputWrapper extends IndexInput {
+public class MockIndexInputWrapper extends FilterIndexInput {
private MockDirectoryWrapper dir;
final String name;
- private IndexInput delegate;
private volatile boolean closed;
// Which MockIndexInputWrapper we were cloned from, or null if we are not a clone:
@@ -37,7 +37,7 @@ public class MockIndexInputWrapper extends IndexInput {
/** Sole constructor */
public MockIndexInputWrapper(
MockDirectoryWrapper dir, String name, IndexInput delegate, MockIndexInputWrapper parent) {
- super("MockIndexInputWrapper(name=" + name + " delegate=" + delegate + ")");
+ super("MockIndexInputWrapper(name=" + name + " delegate=" + delegate + ")", delegate);
// If we are a clone then our parent better not be a clone!
assert parent == null || parent.parent == null;
@@ -45,18 +45,17 @@ public class MockIndexInputWrapper extends IndexInput {
this.parent = parent;
this.name = name;
this.dir = dir;
- this.delegate = delegate;
}
@Override
public void close() throws IOException {
if (closed) {
- delegate.close(); // don't mask double-close bugs
+ in.close(); // don't mask double-close bugs
return;
}
closed = true;
- try (Closeable delegate = this.delegate) {
+ try (Closeable delegate = in) {
// Pending resolution on LUCENE-686 we may want to
// remove the conditional check so we also track that
// all clones get closed:
@@ -86,7 +85,7 @@ public class MockIndexInputWrapper extends IndexInput {
new Exception("clone: " + this).printStackTrace(System.out);
}
dir.inputCloneCount.incrementAndGet();
- IndexInput iiclone = delegate.clone();
+ IndexInput iiclone = in.clone();
MockIndexInputWrapper clone =
new MockIndexInputWrapper(dir, name, iiclone, parent != null ? parent : this);
// Pending resolution on LUCENE-686 we may want to
@@ -113,7 +112,7 @@ public class MockIndexInputWrapper extends IndexInput {
new Exception("slice: " + this).printStackTrace(System.out);
}
dir.inputCloneCount.incrementAndGet();
- IndexInput slice = delegate.slice(sliceDescription, offset, length);
+ IndexInput slice = in.slice(sliceDescription, offset, length);
MockIndexInputWrapper clone =
new MockIndexInputWrapper(dir, sliceDescription, slice, parent != null ? parent : this);
return clone;
@@ -122,91 +121,91 @@ public class MockIndexInputWrapper extends IndexInput {
@Override
public long getFilePointer() {
ensureOpen();
- return delegate.getFilePointer();
+ return in.getFilePointer();
}
@Override
public void seek(long pos) throws IOException {
ensureOpen();
- delegate.seek(pos);
+ in.seek(pos);
}
@Override
public long length() {
ensureOpen();
- return delegate.length();
+ return in.length();
}
@Override
public byte readByte() throws IOException {
ensureOpen();
- return delegate.readByte();
+ return in.readByte();
}
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
ensureOpen();
- delegate.readBytes(b, offset, len);
+ in.readBytes(b, offset, len);
}
@Override
public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
ensureOpen();
- delegate.readBytes(b, offset, len, useBuffer);
+ in.readBytes(b, offset, len, useBuffer);
}
@Override
public void readFloats(float[] floats, int offset, int len) throws IOException {
ensureOpen();
- delegate.readFloats(floats, offset, len);
+ in.readFloats(floats, offset, len);
}
@Override
public short readShort() throws IOException {
ensureOpen();
- return delegate.readShort();
+ return in.readShort();
}
@Override
public int readInt() throws IOException {
ensureOpen();
- return delegate.readInt();
+ return in.readInt();
}
@Override
public long readLong() throws IOException {
ensureOpen();
- return delegate.readLong();
+ return in.readLong();
}
@Override
public String readString() throws IOException {
ensureOpen();
- return delegate.readString();
+ return in.readString();
}
@Override
public int readVInt() throws IOException {
ensureOpen();
- return delegate.readVInt();
+ return in.readVInt();
}
@Override
public long readVLong() throws IOException {
ensureOpen();
- return delegate.readVLong();
+ return in.readVLong();
}
@Override
public int readZInt() throws IOException {
ensureOpen();
- return delegate.readZInt();
+ return in.readZInt();
}
@Override
public long readZLong() throws IOException {
ensureOpen();
- return delegate.readZLong();
+ return in.readZLong();
}
@Override
@@ -218,17 +217,17 @@ public class MockIndexInputWrapper extends IndexInput {
@Override
public Map<String, String> readMapOfStrings() throws IOException {
ensureOpen();
- return delegate.readMapOfStrings();
+ return in.readMapOfStrings();
}
@Override
public Set<String> readSetOfStrings() throws IOException {
ensureOpen();
- return delegate.readSetOfStrings();
+ return in.readSetOfStrings();
}
@Override
public String toString() {
- return "MockIndexInputWrapper(" + delegate + ")";
+ return "MockIndexInputWrapper(" + in + ")";
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexOutputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexOutputWrapper.java
index 005542a2e90..0e76492f093 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexOutputWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexOutputWrapper.java
@@ -20,6 +20,7 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.FilterIndexOutput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -27,20 +28,18 @@ import org.apache.lucene.tests.util.LuceneTestCase;
* Used to create an output stream that will throw an IOException on fake disk full, track max disk
* space actually used, and maybe throw random IOExceptions.
*/
-public class MockIndexOutputWrapper extends IndexOutput {
+public class MockIndexOutputWrapper extends FilterIndexOutput {
private MockDirectoryWrapper dir;
- private final IndexOutput delegate;
private boolean first = true;
final String name;
byte[] singleByte = new byte[1];
/** Construct an empty output buffer. */
- public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput delegate, String name) {
- super("MockIndexOutputWrapper(" + delegate + ")", delegate.getName());
+ public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput out, String name) {
+ super("MockIndexOutputWrapper(" + out + ")", out.getName(), out);
this.dir = dir;
this.name = name;
- this.delegate = delegate;
}
private void checkCrashed() throws IOException {
@@ -67,9 +66,9 @@ public class MockIndexOutputWrapper extends IndexOutput {
if (freeSpace > 0) {
realUsage += freeSpace;
if (b != null) {
- delegate.writeBytes(b, offset, (int) freeSpace);
+ out.writeBytes(b, offset, (int) freeSpace);
} else {
- delegate.copyBytes(in, (int) freeSpace);
+ out.copyBytes(in, (int) freeSpace);
}
}
if (realUsage > dir.maxUsedSize) {
@@ -81,7 +80,7 @@ public class MockIndexOutputWrapper extends IndexOutput {
+ " bytes when writing "
+ name
+ " (file length="
- + delegate.getFilePointer();
+ + out.getFilePointer();
if (freeSpace > 0) {
message += "; wrote " + freeSpace + " of " + len + " bytes";
}
@@ -99,13 +98,13 @@ public class MockIndexOutputWrapper extends IndexOutput {
@Override
public void close() throws IOException {
if (closed) {
- delegate.close(); // don't mask double-close bugs
+ out.close(); // don't mask double-close bugs
return;
}
closed = true;
- try (Closeable delegate = this.delegate) {
- assert delegate != null;
+ try (Closeable out = this.out) {
+ assert out != null;
dir.maybeThrowDeterministicException();
} finally {
dir.removeIndexOutput(this, name);
@@ -140,11 +139,11 @@ public class MockIndexOutputWrapper extends IndexOutput {
if (dir.randomState.nextInt(200) == 0) {
final int half = len / 2;
- delegate.writeBytes(b, offset, half);
+ out.writeBytes(b, offset, half);
Thread.yield();
- delegate.writeBytes(b, offset + half, len - half);
+ out.writeBytes(b, offset + half, len - half);
} else {
- delegate.writeBytes(b, offset, len);
+ out.writeBytes(b, offset, len);
}
dir.maybeThrowDeterministicException();
@@ -158,8 +157,8 @@ public class MockIndexOutputWrapper extends IndexOutput {
}
@Override
- public long getFilePointer() {
- return delegate.getFilePointer();
+ public void writeBytes(byte[] b, int len) throws IOException {
+ writeBytes(b, 0, len);
}
@Override
@@ -168,17 +167,7 @@ public class MockIndexOutputWrapper extends IndexOutput {
checkCrashed();
checkDiskFull(null, 0, input, numBytes);
- delegate.copyBytes(input, numBytes);
+ out.copyBytes(input, numBytes);
dir.maybeThrowDeterministicException();
}
-
- @Override
- public long getChecksum() throws IOException {
- return delegate.getChecksum();
- }
-
- @Override
- public String toString() {
- return "MockIndexOutputWrapper(" + delegate + ")";
- }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/ThrottledIndexOutput.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/ThrottledIndexOutput.java
index 013bdc21d3d..6e29e79ad69 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/ThrottledIndexOutput.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/ThrottledIndexOutput.java
@@ -18,15 +18,14 @@ package org.apache.lucene.tests.util;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.FilterIndexOutput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ThreadInterruptedException;
/** Intentionally slow IndexOutput for testing. */
-public class ThrottledIndexOutput extends IndexOutput {
+public class ThrottledIndexOutput extends FilterIndexOutput {
public static final int DEFAULT_MIN_WRITTEN_BYTES = 1024;
private final int bytesPerSecond;
- private IndexOutput delegate;
private long flushDelayMillis;
private long closeDelayMillis;
private long seekDelayMillis;
@@ -35,29 +34,24 @@ public class ThrottledIndexOutput extends IndexOutput {
private long timeElapsed;
private final byte[] bytes = new byte[1];
- public ThrottledIndexOutput newFromDelegate(IndexOutput output) {
+ public ThrottledIndexOutput newFromDelegate(IndexOutput out) {
return new ThrottledIndexOutput(
- bytesPerSecond,
- flushDelayMillis,
- closeDelayMillis,
- seekDelayMillis,
- minBytesWritten,
- output);
+ bytesPerSecond, flushDelayMillis, closeDelayMillis, seekDelayMillis, minBytesWritten, out);
}
- public ThrottledIndexOutput(int bytesPerSecond, long delayInMillis, IndexOutput delegate) {
+ public ThrottledIndexOutput(int bytesPerSecond, long delayInMillis, IndexOutput out) {
this(
bytesPerSecond,
delayInMillis,
delayInMillis,
delayInMillis,
DEFAULT_MIN_WRITTEN_BYTES,
- delegate);
+ out);
}
public ThrottledIndexOutput(
- int bytesPerSecond, long delays, int minBytesWritten, IndexOutput delegate) {
- this(bytesPerSecond, delays, delays, delays, minBytesWritten, delegate);
+ int bytesPerSecond, long delays, int minBytesWritten, IndexOutput out) {
+ this(bytesPerSecond, delays, delays, delays, minBytesWritten, out);
}
public static final int mBitsToBytes(int mbits) {
@@ -70,10 +64,9 @@ public class ThrottledIndexOutput extends IndexOutput {
long closeDelayMillis,
long seekDelayMillis,
long minBytesWritten,
- IndexOutput delegate) {
- super("ThrottledIndexOutput(" + delegate + ")", delegate == null ? "n/a" : delegate.getName());
+ IndexOutput out) {
+ super("ThrottledIndexOutput(" + out + ")", out == null ? "n/a" : out.getName(), out);
assert bytesPerSecond > 0;
- this.delegate = delegate;
this.bytesPerSecond = bytesPerSecond;
this.flushDelayMillis = flushDelayMillis;
this.closeDelayMillis = closeDelayMillis;
@@ -86,15 +79,10 @@ public class ThrottledIndexOutput extends IndexOutput {
try {
sleep(closeDelayMillis + getDelay(true));
} finally {
- delegate.close();
+ out.close();
}
}
- @Override
- public long getFilePointer() {
- return delegate.getFilePointer();
- }
-
@Override
public void writeByte(byte b) throws IOException {
bytes[0] = b;
@@ -107,7 +95,7 @@ public class ThrottledIndexOutput extends IndexOutput {
// TODO: sometimes, write only half the bytes, then
// sleep, then 2nd half, then sleep, so we sometimes
// interrupt having only written not all bytes
- delegate.writeBytes(b, offset, length);
+ out.writeBytes(b, offset, length);
timeElapsed += System.nanoTime() - before;
pendingBytes += length;
sleep(getDelay(false));
@@ -127,7 +115,7 @@ public class ThrottledIndexOutput extends IndexOutput {
return 0;
}
- private static final void sleep(long ms) {
+ private static void sleep(long ms) {
if (ms <= 0) {
return;
}
@@ -137,14 +125,4 @@ public class ThrottledIndexOutput extends IndexOutput {
throw new ThreadInterruptedException(e);
}
}
-
- @Override
- public void copyBytes(DataInput input, long numBytes) throws IOException {
- delegate.copyBytes(input, numBytes);
- }
-
- @Override
- public long getChecksum() throws IOException {
- return delegate.getChecksum();
- }
}