You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2020/08/14 00:57:45 UTC
[accumulo] branch 1.10 updated: Support IteratorEnvironment in test
harness (#1681)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new c581c43 Support IteratorEnvironment in test harness (#1681)
c581c43 is described below
commit c581c43c6baaac758ed0f2c5cd8cab60be12f60e
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Thu Aug 13 20:57:35 2020 -0400
Support IteratorEnvironment in test harness (#1681)
* Adds the ability to provide a custom IteratorEnvironment to the
iterator-test-harness
* This fixes a test failure in SummingCombinerTest caused by recent
changes to Combiner in #1680
---
.../accumulo/iteratortest/IteratorTestInput.java | 41 ++++++++++++++++++++--
.../iteratortest/testcases/DeepCopyTestCase.java | 5 ++-
.../testcases/IsolatedDeepCopiesTestCase.java | 18 +++++-----
.../testcases/MultipleHasTopCalls.java | 3 +-
.../iteratortest/testcases/ReSeekTestCase.java | 5 ++-
.../iteratortest/testcases/YieldingTestCase.java | 3 +-
.../test/iterator/SummingCombinerTest.java | 10 +++++-
7 files changed, 62 insertions(+), 23 deletions(-)
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java
index 617cb46..32f0a87 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java
@@ -29,7 +29,9 @@ import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
/**
* The necessary user-input to invoke a test on a {@link SortedKeyValueIterator}.
@@ -42,9 +44,10 @@ public class IteratorTestInput {
private final Collection<ByteSequence> families;
private final boolean inclusive;
private final SortedMap<Key,Value> input;
+ private final IteratorEnvironment iteratorEnvironment;
/**
- * Construct an instance of the test input.
+ * Construct an instance of the test input, using {@link SimpleIteratorEnvironment}.
*
* @param iteratorClass
* The class for the iterator to test.
@@ -59,7 +62,31 @@ public class IteratorTestInput {
*/
public IteratorTestInput(Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass,
Map<String,String> iteratorOptions, Range range, SortedMap<Key,Value> input) {
- this(iteratorClass, iteratorOptions, range, input, Collections.<ByteSequence>emptySet(), false);
+ this(iteratorClass, iteratorOptions, range, input, Collections.<ByteSequence>emptySet(), false,
+ new SimpleIteratorEnvironment());
+ }
+
+ /**
+ * Construct an instance of the test input.
+ *
+ * @param iteratorClass
+ * The class for the iterator to test.
+ * @param iteratorOptions
+ * Options, if any, to provide to the iterator ({@link IteratorSetting}'s Map of
+ * properties).
+ * @param range
+ * The Range of data to query ({@link Scanner#setRange(Range)}). By default no column
+ * families filter is specified.
+ * @param input
+ * A sorted collection of Key-Value pairs acting as the table.
+ * @param iterEnv
+ * A provided {@link IteratorEnvironment}.
+ */
+ public IteratorTestInput(Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass,
+ Map<String,String> iteratorOptions, Range range, SortedMap<Key,Value> input,
+ IteratorEnvironment iterEnv) {
+ this(iteratorClass, iteratorOptions, range, input, Collections.<ByteSequence>emptySet(), false,
+ requireNonNull(iterEnv));
}
/**
@@ -78,10 +105,13 @@ public class IteratorTestInput {
* Column families passed to {@link SortedKeyValueIterator#seek}.
* @param inclusive
* Whether the families are inclusive or exclusive.
+ * @param iterEnv
+ * An optional provided {@link IteratorEnvironment}. {@link SimpleIteratorEnvironment}
+ * will be used if null.
*/
public IteratorTestInput(Class<? extends SortedKeyValueIterator<Key,Value>> iteratorClass,
Map<String,String> iteratorOptions, Range range, SortedMap<Key,Value> input,
- Collection<ByteSequence> families, boolean inclusive) {
+ Collection<ByteSequence> families, boolean inclusive, IteratorEnvironment iterEnv) {
// Already immutable
this.iteratorClass = requireNonNull(iteratorClass);
// Make it immutable to the test
@@ -92,6 +122,7 @@ public class IteratorTestInput {
this.input = Collections.unmodifiableSortedMap((requireNonNull(input)));
this.families = Collections.unmodifiableCollection(requireNonNull(families));
this.inclusive = inclusive;
+ this.iteratorEnvironment = iterEnv == null ? new SimpleIteratorEnvironment() : iterEnv;
}
public Class<? extends SortedKeyValueIterator<Key,Value>> getIteratorClass() {
@@ -118,6 +149,10 @@ public class IteratorTestInput {
return input;
}
+ public IteratorEnvironment getIteratorEnvironment() {
+ return iteratorEnvironment;
+ }
+
@Override
public String toString() {
return "[iteratorClass=" + iteratorClass + ", iteratorOptions=" + iteratorOptions + ", range="
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java
index a5c9abd..965e0ef 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/DeepCopyTestCase.java
@@ -25,7 +25,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.iteratortest.IteratorTestInput;
import org.apache.accumulo.iteratortest.IteratorTestOutput;
import org.apache.accumulo.iteratortest.IteratorTestUtil;
-import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
/**
* Test case that verifies that an iterator can use the generated instance from {@code deepCopy}.
@@ -38,9 +37,9 @@ public class DeepCopyTestCase extends OutputVerifyingTestCase {
final SortedKeyValueIterator<Key,Value> source = IteratorTestUtil.createSource(testInput);
try {
- skvi.init(source, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
+ skvi.init(source, testInput.getIteratorOptions(), testInput.getIteratorEnvironment());
- SortedKeyValueIterator<Key,Value> copy = skvi.deepCopy(new SimpleIteratorEnvironment());
+ SortedKeyValueIterator<Key,Value> copy = skvi.deepCopy(testInput.getIteratorEnvironment());
copy.seek(testInput.getRange(), testInput.getFamilies(), testInput.isInclusive());
return new IteratorTestOutput(consume(copy));
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java
index 32c9c4e..1cf199f 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/IsolatedDeepCopiesTestCase.java
@@ -27,11 +27,11 @@ import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.iteratortest.IteratorTestInput;
import org.apache.accumulo.iteratortest.IteratorTestOutput;
import org.apache.accumulo.iteratortest.IteratorTestUtil;
-import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,10 +49,11 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase {
final SortedKeyValueIterator<Key,Value> source = IteratorTestUtil.createSource(testInput);
try {
- skvi.init(source, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
+ IteratorEnvironment iteratorEnvironment = testInput.getIteratorEnvironment();
+ skvi.init(source, testInput.getIteratorOptions(), iteratorEnvironment);
- SortedKeyValueIterator<Key,Value> copy1 = skvi.deepCopy(new SimpleIteratorEnvironment());
- SortedKeyValueIterator<Key,Value> copy2 = copy1.deepCopy(new SimpleIteratorEnvironment());
+ SortedKeyValueIterator<Key,Value> copy1 = skvi.deepCopy(iteratorEnvironment);
+ SortedKeyValueIterator<Key,Value> copy2 = copy1.deepCopy(iteratorEnvironment);
Range seekRange = testInput.getRange();
Collection<ByteSequence> seekColumnFamilies = testInput.getFamilies();
@@ -63,7 +64,7 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase {
copy2.seek(testInput.getRange(), seekColumnFamilies, seekInclusive);
TreeMap<Key,Value> output = consumeMany(new ArrayList<>(Arrays.asList(skvi, copy1, copy2)),
- seekRange, seekColumnFamilies, seekInclusive);
+ seekRange, seekColumnFamilies, seekInclusive, iteratorEnvironment);
return new IteratorTestOutput(output);
} catch (IOException e) {
@@ -72,16 +73,15 @@ public class IsolatedDeepCopiesTestCase extends OutputVerifyingTestCase {
}
TreeMap<Key,Value> consumeMany(Collection<SortedKeyValueIterator<Key,Value>> iterators,
- Range range, Collection<ByteSequence> seekColumnFamilies, boolean seekInclusive)
- throws IOException {
+ Range range, Collection<ByteSequence> seekColumnFamilies, boolean seekInclusive,
+ IteratorEnvironment iterEnv) throws IOException {
TreeMap<Key,Value> data = new TreeMap<>();
// All of the copies should have consistent results from concurrent use
while (allHasTop(iterators)) {
// occasionally deep copy one of the existing iterators
if (random.nextInt(3) == 0) {
log.debug("Deep-copying and re-seeking an iterator");
- SortedKeyValueIterator<Key,Value> newcopy =
- getRandomElement(iterators).deepCopy(new SimpleIteratorEnvironment());
+ SortedKeyValueIterator<Key,Value> newcopy = getRandomElement(iterators).deepCopy(iterEnv);
newcopy.seek(
new Range(getTopKey(iterators), true, range.getEndKey(), range.isEndKeyInclusive()),
seekColumnFamilies, seekInclusive);
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java
index 8974ce3..257b661 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/MultipleHasTopCalls.java
@@ -26,7 +26,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.iteratortest.IteratorTestInput;
import org.apache.accumulo.iteratortest.IteratorTestOutput;
import org.apache.accumulo.iteratortest.IteratorTestUtil;
-import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
/**
* TestCase which asserts that multiple calls to {@link SortedKeyValueIterator#hasTop()} should not
@@ -50,7 +49,7 @@ public class MultipleHasTopCalls extends OutputVerifyingTestCase {
final SortedKeyValueIterator<Key,Value> source = IteratorTestUtil.createSource(testInput);
try {
- skvi.init(source, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
+ skvi.init(source, testInput.getIteratorOptions(), testInput.getIteratorEnvironment());
skvi.seek(testInput.getRange(), testInput.getFamilies(), testInput.isInclusive());
return new IteratorTestOutput(consume(skvi));
} catch (IOException e) {
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java
index b447275..bf1dd24 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/ReSeekTestCase.java
@@ -29,7 +29,6 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.iteratortest.IteratorTestInput;
import org.apache.accumulo.iteratortest.IteratorTestOutput;
import org.apache.accumulo.iteratortest.IteratorTestUtil;
-import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,7 +56,7 @@ public class ReSeekTestCase extends OutputVerifyingTestCase {
final SortedKeyValueIterator<Key,Value> source = IteratorTestUtil.createSource(testInput);
try {
- skvi.init(source, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
+ skvi.init(source, testInput.getIteratorOptions(), testInput.getIteratorEnvironment());
skvi.seek(testInput.getRange(), testInput.getFamilies(), testInput.isInclusive());
return new IteratorTestOutput(consume(skvi, testInput));
} catch (IOException e) {
@@ -94,7 +93,7 @@ public class ReSeekTestCase extends OutputVerifyingTestCase {
final SortedKeyValueIterator<Key,Value> sourceCopy =
IteratorTestUtil.createSource(testInput);
- skvi.init(sourceCopy, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
+ skvi.init(sourceCopy, testInput.getIteratorOptions(), testInput.getIteratorEnvironment());
// The new range, resume where we left off (non-inclusive), with same families filter
final Range newRange =
diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/YieldingTestCase.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/YieldingTestCase.java
index 32458d5..392770d 100644
--- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/YieldingTestCase.java
+++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/testcases/YieldingTestCase.java
@@ -28,7 +28,6 @@ import org.apache.accumulo.core.iterators.YieldingKeyValueIterator;
import org.apache.accumulo.iteratortest.IteratorTestInput;
import org.apache.accumulo.iteratortest.IteratorTestOutput;
import org.apache.accumulo.iteratortest.IteratorTestUtil;
-import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
/**
* Test case that verifies that an iterator works correctly with the yielding api. Note that most
@@ -45,7 +44,7 @@ public class YieldingTestCase extends OutputVerifyingTestCase {
final SortedKeyValueIterator<Key,Value> source = IteratorTestUtil.createSource(testInput);
try {
- skvi.init(source, testInput.getIteratorOptions(), new SimpleIteratorEnvironment());
+ skvi.init(source, testInput.getIteratorOptions(), testInput.getIteratorEnvironment());
YieldCallback<Key> yield = new YieldCallback<>();
if (skvi instanceof YieldingKeyValueIterator) {
diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java
index f4939d6..aca1f39 100644
--- a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java
@@ -29,11 +29,13 @@ import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Combiner;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.LongCombiner;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.iteratortest.IteratorTestCaseFinder;
import org.apache.accumulo.iteratortest.IteratorTestInput;
import org.apache.accumulo.iteratortest.IteratorTestOutput;
+import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment;
import org.apache.accumulo.iteratortest.junit4.BaseJUnit4IteratorTest;
import org.apache.accumulo.iteratortest.testcases.IteratorTestCase;
import org.junit.runners.Parameterized.Parameters;
@@ -118,8 +120,14 @@ public class SummingCombinerTest extends BaseJUnit4IteratorTest {
IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
LongCombiner.setEncodingType(setting, LongCombiner.Type.STRING);
Combiner.setCombineAllColumns(setting, true);
+ Combiner.setReduceOnFullCompactionOnly(setting, false);
return new IteratorTestInput(SummingCombiner.class, setting.getOptions(), new Range(),
- INPUT_DATA);
+ INPUT_DATA, new SimpleIteratorEnvironment() {
+ @Override
+ public IteratorScope getIteratorScope() {
+ return IteratorScope.majc;
+ }
+ });
}
private static IteratorTestOutput getIteratorOutput() {