You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2016/11/29 14:01:50 UTC
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8
Conflicts:
core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cd70d781
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cd70d781
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cd70d781
Branch: refs/heads/1.8
Commit: cd70d7815afad9ae7d0f78c5fa916833b274690b
Parents: 43b740b a4a4539
Author: Mike Miller <mm...@apache.org>
Authored: Tue Nov 22 14:58:58 2016 -0500
Committer: Mike Miller <mm...@apache.org>
Committed: Tue Nov 22 14:58:58 2016 -0500
----------------------------------------------------------------------
.../apache/accumulo/core/data/RangeTest.java | 428 ++++++++++---------
.../core/file/rfile/BlockIndexTest.java | 22 +-
.../core/file/rfile/CreateCompatTestFile.java | 20 +-
.../core/file/rfile/MultiThreadedRFileTest.java | 8 +-
.../core/file/rfile/RFileMetricsTest.java | 78 ++--
.../accumulo/core/file/rfile/RFileTest.java | 416 +++++++++---------
.../core/file/rfile/RelativeKeyTest.java | 14 +-
.../core/iterators/AggregatingIteratorTest.java | 152 +++----
.../core/iterators/FirstEntryInRowTest.java | 46 +-
.../core/iterators/IteratorUtilTest.java | 38 +-
.../ColumnFamilySkippingIteratorTest.java | 44 +-
.../core/iterators/system/ColumnFilterTest.java | 30 +-
.../iterators/system/DeletingIteratorTest.java | 62 +--
.../iterators/system/MultiIteratorTest.java | 125 +++---
.../system/SourceSwitchingIteratorTest.java | 45 +-
.../iterators/user/BigDecimalCombinerTest.java | 20 +-
.../iterators/user/ColumnSliceFilterTest.java | 18 +-
.../core/iterators/user/CombinerTest.java | 220 +++++-----
.../iterators/user/IndexedDocIteratorTest.java | 4 +-
.../core/iterators/user/RegExFilterTest.java | 12 +-
.../iterators/user/RowDeletingIteratorTest.java | 80 ++--
.../tserver/CheckTabletMetadataTest.java | 14 +-
.../accumulo/tserver/InMemoryMapTest.java | 178 ++++----
.../accumulo/test/NativeMapPerformanceTest.java | 12 +-
.../accumulo/test/functional/NativeMapIT.java | 79 ++--
25 files changed, 1086 insertions(+), 1079 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
index c4837fe,115aa7a..fe7f1f8
--- a/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java
@@@ -191,30 -191,31 +191,31 @@@ public class RangeTest extends TestCas
public void testMergeOverlapping22() {
- Range ke1 = new KeyExtent(new Text("tab1"), new Text("Bank"), null).toMetadataRange();
- Range ke2 = new KeyExtent(new Text("tab1"), new Text("Fails"), new Text("Bank")).toMetadataRange();
- Range ke3 = new KeyExtent(new Text("tab1"), new Text("Sam"), new Text("Fails")).toMetadataRange();
- Range ke4 = new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Sam")).toMetadataRange();
- Range ke5 = new KeyExtent(new Text("tab1"), null, new Text("bails")).toMetadataRange();
+ Range ke1 = new KeyExtent("tab1", new Text("Bank"), null).toMetadataRange();
+ Range ke2 = new KeyExtent("tab1", new Text("Fails"), new Text("Bank")).toMetadataRange();
+ Range ke3 = new KeyExtent("tab1", new Text("Sam"), new Text("Fails")).toMetadataRange();
+ Range ke4 = new KeyExtent("tab1", new Text("bails"), new Text("Sam")).toMetadataRange();
+ Range ke5 = new KeyExtent("tab1", null, new Text("bails")).toMetadataRange();
- List<Range> rl = nrl(ke1, ke2, ke3, ke4, ke5);
- List<Range> expected = nrl(new KeyExtent("tab1", null, null).toMetadataRange());
+ List<Range> rl = newRangeList(ke1, ke2, ke3, ke4, ke5);
- List<Range> expected = newRangeList(new KeyExtent(new Text("tab1"), null, null).toMetadataRange());
++ List<Range> expected = newRangeList(new KeyExtent("tab1", null, null).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
- rl = nrl(ke1, ke2, ke4, ke5);
- expected = nrl(new KeyExtent("tab1", new Text("Fails"), null).toMetadataRange(), new KeyExtent("tab1", null, new Text("Sam")).toMetadataRange());
+ rl = newRangeList(ke1, ke2, ke4, ke5);
- expected = newRangeList(new KeyExtent(new Text("tab1"), new Text("Fails"), null).toMetadataRange(),
- new KeyExtent(new Text("tab1"), null, new Text("Sam")).toMetadataRange());
++ expected = newRangeList(new KeyExtent("tab1", new Text("Fails"), null).toMetadataRange(), new KeyExtent("tab1", null, new Text("Sam")).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
- rl = nrl(ke2, ke3, ke4, ke5);
- expected = nrl(new KeyExtent("tab1", null, new Text("Bank")).toMetadataRange());
+ rl = newRangeList(ke2, ke3, ke4, ke5);
- expected = newRangeList(new KeyExtent(new Text("tab1"), null, new Text("Bank")).toMetadataRange());
++ expected = newRangeList(new KeyExtent("tab1", null, new Text("Bank")).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
- rl = nrl(ke1, ke2, ke3, ke4);
- expected = nrl(new KeyExtent("tab1", new Text("bails"), null).toMetadataRange());
+ rl = newRangeList(ke1, ke2, ke3, ke4);
- expected = newRangeList(new KeyExtent(new Text("tab1"), new Text("bails"), null).toMetadataRange());
++ expected = newRangeList(new KeyExtent("tab1", new Text("bails"), null).toMetadataRange());
check(Range.mergeOverlapping(rl), expected);
- rl = nrl(ke2, ke3, ke4);
- expected = nrl(new KeyExtent("tab1", new Text("bails"), new Text("Bank")).toMetadataRange());
+ rl = newRangeList(ke2, ke3, ke4);
- expected = newRangeList(new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Bank")).toMetadataRange());
++ expected = newRangeList(new KeyExtent("tab1", new Text("bails"), new Text("Bank")).toMetadataRange());
++
check(Range.mergeOverlapping(rl), expected);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
index e7c8b46,eec4e71..34d01ad
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java
@@@ -56,20 -56,20 +56,20 @@@ public class CreateCompatTestFile
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
- CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", conf, AccumuloConfiguration.getDefaultConfiguration());
+ CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", null, conf, AccumuloConfiguration.getDefaultConfiguration());
RFile.Writer writer = new RFile.Writer(_cbw, 1000);
- writer.startNewLocalityGroup("lg1", ncfs(nf("cf_", 1), nf("cf_", 2)));
+ writer.startNewLocalityGroup("lg1", newColFamSequence(formatStr("cf_", 1), formatStr("cf_", 2)));
for (int i = 0; i < 1000; i++) {
- writer.append(nk(nf("r_", i), nf("cf_", 1), nf("cq_", 0), "", 1000 - i), nv(i + ""));
- writer.append(nk(nf("r_", i), nf("cf_", 2), nf("cq_", 0), "", 1000 - i), nv(i + ""));
+ writer.append(newKey(formatStr("r_", i), formatStr("cf_", 1), formatStr("cq_", 0), "", 1000 - i), newValue(i + ""));
+ writer.append(newKey(formatStr("r_", i), formatStr("cf_", 2), formatStr("cq_", 0), "", 1000 - i), newValue(i + ""));
}
- writer.startNewLocalityGroup("lg2", ncfs(nf("cf_", 3)));
+ writer.startNewLocalityGroup("lg2", newColFamSequence(formatStr("cf_", 3)));
for (int i = 0; i < 1000; i++) {
- writer.append(nk(nf("r_", i), nf("cf_", 3), nf("cq_", 0), "", 1000 - i), nv(i + ""));
+ writer.append(newKey(formatStr("r_", i), formatStr("cf_", 3), formatStr("cq_", 0), "", 1000 - i), newValue(i + ""));
}
writer.startDefaultLocalityGroup();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileMetricsTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
index 3833564,96f97e7..aff1a8f
--- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
@@@ -378,18 -338,18 +378,18 @@@ public class RFileTest
for (int cv = 'A'; cv < 'A' + 4; cv++) {
String cvS = "" + (char) cv;
for (int ts = 4; ts > 0; ts--) {
- Key k = nk(rowS, cfS, cqS, cvS, ts);
+ Key k = newKey(rowS, cfS, cqS, cvS, ts);
// check below ensures when all key sizes are same more than one index block is created
- assertEquals(27, k.getSize());
+ Assert.assertEquals(27, k.getSize());
k.setDeleted(true);
- Value v = nv("" + val);
+ Value v = newValue("" + val);
trf.writer.append(k, v);
expectedKeys.add(k);
expectedValues.add(v);
- k = nk(rowS, cfS, cqS, cvS, ts);
+ k = newKey(rowS, cfS, cqS, cvS, ts);
- assertEquals(27, k.getSize());
+ Assert.assertEquals(27, k.getSize());
- v = nv("" + val);
+ v = newValue("" + val);
trf.writer.append(k, v);
expectedKeys.add(k);
expectedValues.add(v);
@@@ -531,39 -491,39 +531,39 @@@
trf.openWriter();
- trf.writer.append(nk("r1", "cf1", "cq1", "L1", 55), nv("foo1"));
+ trf.writer.append(newKey("r1", "cf1", "cq1", "L1", 55), newValue("foo1"));
try {
- trf.writer.append(nk("r0", "cf1", "cq1", "L1", 55), nv("foo1"));
+ trf.writer.append(newKey("r0", "cf1", "cq1", "L1", 55), newValue("foo1"));
assertFalse(true);
- } catch (IllegalStateException ioe) {
+ } catch (IllegalArgumentException ioe) {
}
try {
- trf.writer.append(nk("r1", "cf0", "cq1", "L1", 55), nv("foo1"));
+ trf.writer.append(newKey("r1", "cf0", "cq1", "L1", 55), newValue("foo1"));
assertFalse(true);
- } catch (IllegalStateException ioe) {
+ } catch (IllegalArgumentException ioe) {
}
try {
- trf.writer.append(nk("r1", "cf1", "cq0", "L1", 55), nv("foo1"));
+ trf.writer.append(newKey("r1", "cf1", "cq0", "L1", 55), newValue("foo1"));
assertFalse(true);
- } catch (IllegalStateException ioe) {
+ } catch (IllegalArgumentException ioe) {
}
try {
- trf.writer.append(nk("r1", "cf1", "cq1", "L0", 55), nv("foo1"));
+ trf.writer.append(newKey("r1", "cf1", "cq1", "L0", 55), newValue("foo1"));
assertFalse(true);
- } catch (IllegalStateException ioe) {
+ } catch (IllegalArgumentException ioe) {
}
try {
- trf.writer.append(nk("r1", "cf1", "cq1", "L1", 56), nv("foo1"));
+ trf.writer.append(newKey("r1", "cf1", "cq1", "L1", 56), newValue("foo1"));
assertFalse(true);
- } catch (IllegalStateException ioe) {
+ } catch (IllegalArgumentException ioe) {
}
}
@@@ -646,38 -606,44 +646,38 @@@
trf.openReader();
// test that has top returns false when end of range reached
- trf.iter.seek(new Range(nk(nf("r_", 3), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false);
- trf.iter.seek(new Range(newKey(formatString("r_", 3), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4), "cf1", "cq1", "L1", 55), false),
- EMPTY_COL_FAMS, false);
++ trf.iter.seek(new Range(newKey(formatString("r_", 3), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false);
assertTrue(trf.iter.hasTop());
- assertTrue(trf.iter.getTopKey().equals(nk(nf("r_", 3), "cf1", "cq1", "L1", 55)));
- assertEquals(nv("foo" + 3), trf.iter.getTopValue());
+ assertTrue(trf.iter.getTopKey().equals(newKey(formatString("r_", 3), "cf1", "cq1", "L1", 55)));
+ assertEquals(newValue("foo" + 3), trf.iter.getTopValue());
trf.iter.next();
assertFalse(trf.iter.hasTop());
// test seeking to a range that is between two keys, should not return anything
- trf.iter.seek(new Range(nk(nf("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "b", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false);
- trf.iter.seek(new Range(newKey(formatString("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "b", "cf1", "cq1", "L1", 55),
- true), EMPTY_COL_FAMS, false);
++ trf.iter.seek(new Range(newKey(formatString("r_", 4) + "a", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "b", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false);
assertFalse(trf.iter.hasTop());
// test seeking to another range after the previously seeked range, that is between the same two keys in the file
// as the previously seeked range.... this test an optimization on RFile
- trf.iter.seek(new Range(nk(nf("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "d", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false);
- trf.iter.seek(new Range(newKey(formatString("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "d", "cf1", "cq1", "L1", 55),
- true), EMPTY_COL_FAMS, false);
++ trf.iter.seek(new Range(newKey(formatString("r_", 4) + "c", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "d", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false);
assertFalse(trf.iter.hasTop());
- trf.iter.seek(new Range(nk(nf("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, nk(nf("r_", 4) + "f", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false);
- trf.iter.seek(new Range(newKey(formatString("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "f", "cf1", "cq1", "L1", 55),
- true), EMPTY_COL_FAMS, false);
++ trf.iter.seek(new Range(newKey(formatString("r_", 4) + "e", "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 4) + "f", "cf1", "cq1", "L1", 55), true), EMPTY_COL_FAMS, false);
assertFalse(trf.iter.hasTop());
// now ensure we can seek somewhere, that triggering the optimization does not cause any problems
- trf.iter.seek(new Range(nk(nf("r_", 5), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 6), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false);
- trf.iter.seek(new Range(newKey(formatString("r_", 5), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 6), "cf1", "cq1", "L1", 55), false),
- EMPTY_COL_FAMS, false);
++ trf.iter.seek(new Range(newKey(formatString("r_", 5), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 6), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false);
assertTrue(trf.iter.hasTop());
- assertTrue(trf.iter.getTopKey().equals(nk(nf("r_", 5), "cf1", "cq1", "L1", 55)));
- assertEquals(nv("foo" + 5), trf.iter.getTopValue());
+ assertTrue(trf.iter.getTopKey().equals(newKey(formatString("r_", 5), "cf1", "cq1", "L1", 55)));
+ assertEquals(newValue("foo" + 5), trf.iter.getTopValue());
trf.iter.next();
assertFalse(trf.iter.hasTop());
// test seeking to range that is before the beginning of the file
- trf.iter.seek(new Range(nk(nf("r_", 0), "cf1", "cq1", "L1", 55), true, nk(nf("r_", 2), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false);
- trf.iter.seek(new Range(newKey(formatString("r_", 0), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 2), "cf1", "cq1", "L1", 55), false),
- EMPTY_COL_FAMS, false);
++ trf.iter.seek(new Range(newKey(formatString("r_", 0), "cf1", "cq1", "L1", 55), true, newKey(formatString("r_", 2), "cf1", "cq1", "L1", 55), false), EMPTY_COL_FAMS, false);
assertFalse(trf.iter.hasTop());
- assertEquals(nk(nf("r_", 49), "cf1", "cq1", "L1", 55), trf.reader.getLastKey());
+ assertEquals(newKey(formatString("r_", 49), "cf1", "cq1", "L1", 55), trf.reader.getLastKey());
trf.reader.close();
}
@@@ -805,23 -771,23 +805,23 @@@
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2"));
- trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st"));
- trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st"));
+ trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st"));
+ trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st"));
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4"));
- trf.writer.append(nk("0001", "cf3", "buck,john", "", 4), nv("90 Slum st"));
- trf.writer.append(nk("0003", "cf4", "buck,jane", "", 5), nv("09 Slum st"));
+ trf.writer.append(newKey("0001", "cf3", "buck,john", "", 4), newValue("90 Slum st"));
+ trf.writer.append(newKey("0003", "cf4", "buck,jane", "", 5), newValue("09 Slum st"));
trf.writer.close();
trf.openReader();
// scan first loc group
- Range r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ Range r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf1", "cf2"), true);
+ trf.iter.seek(r, ncfs("cf1", "cf2"), true);
assertEquals(1, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -835,8 -801,8 +835,8 @@@
assertFalse(trf.iter.hasTop());
// scan second loc group
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf3", "cf4"), true);
+ trf.iter.seek(r, ncfs("cf3", "cf4"), true);
assertEquals(1, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -873,14 -839,14 +873,14 @@@
assertFalse(trf.iter.hasTop());
// scan no loc groups
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("saint", "dogooder"), true);
+ trf.iter.seek(r, ncfs("saint", "dogooder"), true);
assertEquals(0, trf.reader.getNumLocalityGroupsSeeked());
assertFalse(trf.iter.hasTop());
// scan a subset of second locality group
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf4"), true);
+ trf.iter.seek(r, ncfs("cf4"), true);
assertEquals(1, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -890,8 -856,8 +890,8 @@@
assertFalse(trf.iter.hasTop());
// scan a subset of second locality group
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf3"), true);
+ trf.iter.seek(r, ncfs("cf3"), true);
assertEquals(1, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -901,8 -867,8 +901,8 @@@
assertFalse(trf.iter.hasTop());
// scan subset of first loc group
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf1"), true);
+ trf.iter.seek(r, ncfs("cf1"), true);
assertEquals(1, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -912,8 -878,8 +912,8 @@@
assertFalse(trf.iter.hasTop());
// scan subset of first loc group
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf2"), true);
+ trf.iter.seek(r, ncfs("cf2"), true);
assertEquals(1, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -923,8 -889,8 +923,8 @@@
assertFalse(trf.iter.hasTop());
// scan subset of all loc groups
- r = new Range(nk("0000", "cf1", "doe,john", "", 4), true, nk("0003", "cf4", "buck,jane", "", 5), true);
+ r = new Range(newKey("0000", "cf1", "doe,john", "", 4), true, newKey("0003", "cf4", "buck,jane", "", 5), true);
- trf.iter.seek(r, newColFamByteSequence("cf1", "cf4"), true);
+ trf.iter.seek(r, ncfs("cf1", "cf4"), true);
assertEquals(2, trf.reader.getNumLocalityGroupsSeeked());
assertTrue(trf.iter.hasTop());
@@@ -963,10 -929,10 +963,10 @@@
trf = new TestRFile(conf);
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2"));
- trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st"));
- trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st"));
+ trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st"));
+ trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st"));
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4"));
trf.writer.startDefaultLocalityGroup();
trf.writer.close();
@@@ -988,10 -954,10 +988,10 @@@
trf = new TestRFile(conf);
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2"));
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4"));
- trf.writer.append(nk("0001", "cf3", "buck,john", "", 4), nv("90 Slum st"));
- trf.writer.append(nk("0003", "cf4", "buck,jane", "", 5), nv("09 Slum st"));
+ trf.writer.append(newKey("0001", "cf3", "buck,john", "", 4), newValue("90 Slum st"));
+ trf.writer.append(newKey("0003", "cf4", "buck,jane", "", 5), newValue("09 Slum st"));
trf.writer.startDefaultLocalityGroup();
trf.writer.close();
@@@ -1013,11 -979,11 +1013,11 @@@
trf = new TestRFile(conf);
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2"));
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4"));
trf.writer.startDefaultLocalityGroup();
- trf.writer.append(nk("0007", "good citizen", "q,john", "", 4), nv("70 Apple st"));
- trf.writer.append(nk("0008", "model citizen", "q,jane", "", 5), nv("81 Plum st"));
+ trf.writer.append(newKey("0007", "good citizen", "q,john", "", 4), newValue("70 Apple st"));
+ trf.writer.append(newKey("0008", "model citizen", "q,jane", "", 5), newValue("81 Plum st"));
trf.writer.close();
trf.openReader();
@@@ -1038,13 -1004,13 +1038,13 @@@
trf = new TestRFile(conf);
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2"));
- trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st"));
- trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st"));
+ trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st"));
+ trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st"));
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4"));
trf.writer.startDefaultLocalityGroup();
- trf.writer.append(nk("0007", "good citizen", "q,john", "", 4), nv("70 Apple st"));
- trf.writer.append(nk("0008", "model citizen", "q,jane", "", 5), nv("81 Plum st"));
+ trf.writer.append(newKey("0007", "good citizen", "q,john", "", 4), newValue("70 Apple st"));
+ trf.writer.append(newKey("0008", "model citizen", "q,jane", "", 5), newValue("81 Plum st"));
trf.writer.close();
trf.openReader();
@@@ -1077,17 -1043,17 +1077,17 @@@
TestRFile trf = new TestRFile(conf);
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("3mod10"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("3mod10"));
for (int i = 3; i < 1024; i += 10) {
- trf.writer.append(nk(nf("i", i), "3mod10", "", "", i + 2), nv("" + i));
+ trf.writer.append(newKey(formatString("i", i), "3mod10", "", "", i + 2), newValue("" + i));
}
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("5mod10", "7mod10"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("5mod10", "7mod10"));
for (int i = 5; i < 1024;) {
- trf.writer.append(nk(nf("i", i), "5mod10", "", "", i + 2), nv("" + i));
+ trf.writer.append(newKey(formatString("i", i), "5mod10", "", "", i + 2), newValue("" + i));
i += 2;
- trf.writer.append(nk(nf("i", i), "7mod10", "", "", i + 2), nv("" + i));
+ trf.writer.append(newKey(formatString("i", i), "7mod10", "", "", i + 2), newValue("" + i));
i += 8;
}
@@@ -1183,12 -1149,12 +1183,12 @@@
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("a", "b"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b"));
- trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1"));
+ trf.writer.append(newKey("0007", "a", "cq1", "", 4), newValue("1"));
try {
- trf.writer.append(nk("0009", "c", "cq1", "", 4), nv("1"));
+ trf.writer.append(newKey("0009", "c", "cq1", "", 4), newValue("1"));
assertFalse(true);
} catch (IllegalArgumentException ioe) {
@@@ -1216,9 -1182,9 +1216,9 @@@
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("a", "b"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b"));
- trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1"));
+ trf.writer.append(newKey("0007", "a", "cq1", "", 4), newValue("1"));
trf.writer.startDefaultLocalityGroup();
@@@ -1280,11 -1247,11 +1280,11 @@@
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("a", "b"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("a", "b"));
- trf.writer.append(nk("0007", "a", "cq1", "", 4), nv("1"));
+ trf.writer.append(newKey("0007", "a", "cq1", "", 4), newValue("1"));
try {
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("b", "c"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("b", "c"));
assertFalse(true);
} catch (IllegalArgumentException ioe) {
@@@ -1492,10 -1459,10 +1492,10 @@@
trf.openWriter(false);
trf.openWriter(false);
- trf.writer.startNewLocalityGroup("lg1", newColFamByteSequence("cf1", "cf2"));
+ trf.writer.startNewLocalityGroup("lg1", ncfs("cf1", "cf2"));
- trf.writer.append(nk("0000", "cf1", "doe,john", "", 4), nv("1123 West Left st"));
- trf.writer.append(nk("0002", "cf2", "doe,jane", "", 5), nv("1124 East Right st"));
+ trf.writer.append(newKey("0000", "cf1", "doe,john", "", 4), newValue("1123 West Left st"));
+ trf.writer.append(newKey("0002", "cf2", "doe,jane", "", 5), newValue("1124 East Right st"));
- trf.writer.startNewLocalityGroup("lg2", newColFamByteSequence("cf3", "cf4"));
+ trf.writer.startNewLocalityGroup("lg2", ncfs("cf3", "cf4"));
DataOutputStream dos = trf.writer.createMetaStore("count");
@@@ -1627,9 -1593,9 +1627,9 @@@
for (int start : new int[] {0, 10, 100, 998}) {
for (int cf = 1; cf <= 4; cf++) {
if (start == 0)
- iter.seek(new Range(), ncfs(nf("cf_", cf)), true);
- iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true);
++ iter.seek(new Range(), ncfs(formatString("cf_", cf)), true);
else
- iter.seek(new Range(nf("r_", start), null), ncfs(nf("cf_", cf)), true);
- iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true);
++ iter.seek(new Range(formatString("r_", start), null), ncfs(formatString("cf_", cf)), true);
for (int i = start; i < 1000; i++) {
assertTrue(iter.hasTop());
@@@ -1642,9 -1608,9 +1642,9 @@@
}
if (start == 0)
- iter.seek(new Range(), newColFamByteSequence(), false);
+ iter.seek(new Range(), ncfs(), false);
else
- iter.seek(new Range(nf("r_", start), null), ncfs(), false);
- iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false);
++ iter.seek(new Range(formatString("r_", start), null), ncfs(), false);
for (int i = start; i < 1000; i++) {
for (int cf = 1; cf <= 4; cf++) {
@@@ -1808,294 -1774,6 +1808,294 @@@
conf = null;
}
- private Key nk(int r, int c) {
++ private Key newKey(int r, int c) {
+ String row = String.format("r%06d", r);
+ switch (c) {
+ case 0:
+ return new Key(row, "user", "addr");
+ case 1:
+ return new Key(row, "user", "name");
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
+
- private Value nv(int r, int c) {
++ private Value newValue(int r, int c) {
+ switch (c) {
+ case 0:
+ return new Value(("123" + r + " west st").getBytes());
+ case 1:
+ return new Value(("bob" + r).getBytes());
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
+
+ private static void hash(Hasher hasher, Key key, Value val) {
+ hasher.putBytes(key.getRowData().toArray());
+ hasher.putBytes(key.getColumnFamilyData().toArray());
+ hasher.putBytes(key.getColumnQualifierData().toArray());
+ hasher.putBytes(key.getColumnVisibilityData().toArray());
+ hasher.putLong(key.getTimestamp());
+ hasher.putBoolean(key.isDeleted());
+ hasher.putBytes(val.get());
+ }
+
+ private static void add(TestRFile trf, Key key, Value val, Hasher dataHasher, List<Entry<Key,Value>> sample, Sampler sampler) throws IOException {
+ if (sampler.accept(key)) {
+ sample.add(new AbstractMap.SimpleImmutableEntry<>(key, val));
+ }
+
+ hash(dataHasher, key, val);
+
+ trf.writer.append(key, val);
+ }
+
+ private List<Entry<Key,Value>> toList(SortedKeyValueIterator<Key,Value> sample) throws IOException {
+ ArrayList<Entry<Key,Value>> ret = new ArrayList<>();
+
+ while (sample.hasTop()) {
+ ret.add(new AbstractMap.SimpleImmutableEntry<>(new Key(sample.getTopKey()), new Value(sample.getTopValue())));
+ sample.next();
+ }
+
+ return ret;
+ }
+
+ private void checkSample(SortedKeyValueIterator<Key,Value> sample, List<Entry<Key,Value>> sampleData) throws IOException {
+ checkSample(sample, sampleData, EMPTY_COL_FAMS, false);
+ }
+
+ private void checkSample(SortedKeyValueIterator<Key,Value> sample, List<Entry<Key,Value>> sampleData, Collection<ByteSequence> columnFamilies,
+ boolean inclusive) throws IOException {
+
+ sample.seek(new Range(), columnFamilies, inclusive);
+ Assert.assertEquals(sampleData, toList(sample));
+
+ Random rand = new Random();
+ long seed = rand.nextLong();
+ rand = new Random(seed);
+
+ // randomly seek sample iterator and verify
+ for (int i = 0; i < 33; i++) {
+ Key startKey = null;
+ boolean startInclusive = false;
+ int startIndex = 0;
+
+ Key endKey = null;
+ boolean endInclusive = false;
+ int endIndex = sampleData.size();
+
+ if (rand.nextBoolean()) {
+ startIndex = rand.nextInt(sampleData.size());
+ startKey = sampleData.get(startIndex).getKey();
+ startInclusive = rand.nextBoolean();
+ if (!startInclusive) {
+ startIndex++;
+ }
+ }
+
+ if (startIndex < endIndex && rand.nextBoolean()) {
+ endIndex -= rand.nextInt(endIndex - startIndex);
+ endKey = sampleData.get(endIndex - 1).getKey();
+ endInclusive = rand.nextBoolean();
+ if (!endInclusive) {
+ endIndex--;
+ }
+ } else if (startIndex == endIndex) {
+ endInclusive = rand.nextBoolean();
+ }
+
+ sample.seek(new Range(startKey, startInclusive, endKey, endInclusive), columnFamilies, inclusive);
+ Assert.assertEquals("seed: " + seed, sampleData.subList(startIndex, endIndex), toList(sample));
+ }
+ }
+
+ @Test
+ public void testSample() throws IOException {
+
+ int num = 10000;
+
+ for (int sampleBufferSize : new int[] {1 << 10, 1 << 20}) {
+ // force sample buffer to flush for smaller data
+ RFile.setSampleBufferSize(sampleBufferSize);
+
+ for (int modulus : new int[] {19, 103, 1019}) {
+ Hasher dataHasher = Hashing.md5().newHasher();
+ List<Entry<Key,Value>> sampleData = new ArrayList<>();
+
+ ConfigurationCopy sampleConf = new ConfigurationCopy(conf == null ? AccumuloConfiguration.getDefaultConfiguration() : conf);
+ sampleConf.set(Property.TABLE_SAMPLER, RowSampler.class.getName());
+ sampleConf.set(Property.TABLE_SAMPLER_OPTS + "hasher", "murmur3_32");
+ sampleConf.set(Property.TABLE_SAMPLER_OPTS + "modulus", modulus + "");
+
+ Sampler sampler = SamplerFactory.newSampler(SamplerConfigurationImpl.newSamplerConfig(sampleConf), sampleConf);
+
+ TestRFile trf = new TestRFile(sampleConf);
+
+ trf.openWriter();
+
+ for (int i = 0; i < num; i++) {
- add(trf, nk(i, 0), nv(i, 0), dataHasher, sampleData, sampler);
- add(trf, nk(i, 1), nv(i, 1), dataHasher, sampleData, sampler);
++ add(trf, newKey(i, 0), newValue(i, 0), dataHasher, sampleData, sampler);
++ add(trf, newKey(i, 1), newValue(i, 1), dataHasher, sampleData, sampler);
+ }
+
+ HashCode expectedDataHash = dataHasher.hash();
+
+ trf.closeWriter();
+
+ trf.openReader();
+
+ FileSKVIterator sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf));
+
+ checkSample(sample, sampleData);
+
+ Assert.assertEquals(expectedDataHash, hash(trf.reader));
+
+ SampleIE ie = new SampleIE(SamplerConfigurationImpl.newSamplerConfig(sampleConf).toSamplerConfiguration());
+
+ for (int i = 0; i < 3; i++) {
+ // test opening and closing deep copies a few times.
+ trf.reader.closeDeepCopies();
+
+ sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf));
+ SortedKeyValueIterator<Key,Value> sampleDC1 = sample.deepCopy(ie);
+ SortedKeyValueIterator<Key,Value> sampleDC2 = sample.deepCopy(ie);
+ SortedKeyValueIterator<Key,Value> sampleDC3 = trf.reader.deepCopy(ie);
+ SortedKeyValueIterator<Key,Value> allDC1 = sampleDC1.deepCopy(new SampleIE(null));
+ SortedKeyValueIterator<Key,Value> allDC2 = sample.deepCopy(new SampleIE(null));
+
+ Assert.assertEquals(expectedDataHash, hash(allDC1));
+ Assert.assertEquals(expectedDataHash, hash(allDC2));
+
+ checkSample(sample, sampleData);
+ checkSample(sampleDC1, sampleData);
+ checkSample(sampleDC2, sampleData);
+ checkSample(sampleDC3, sampleData);
+ }
+
+ trf.reader.closeDeepCopies();
+
+ trf.closeReader();
+ }
+ }
+ }
+
+ private HashCode hash(SortedKeyValueIterator<Key,Value> iter) throws IOException {
+ Hasher dataHasher = Hashing.md5().newHasher();
+ iter.seek(new Range(), EMPTY_COL_FAMS, false);
+ while (iter.hasTop()) {
+ hash(dataHasher, iter.getTopKey(), iter.getTopValue());
+ iter.next();
+ }
+
+ return dataHasher.hash();
+ }
+
+ @Test
+ public void testSampleLG() throws IOException {
+
+ int num = 5000;
+
+ for (int sampleBufferSize : new int[] {1 << 10, 1 << 20}) {
+ // force sample buffer to flush for smaller data
+ RFile.setSampleBufferSize(sampleBufferSize);
+
+ for (int modulus : new int[] {19, 103, 1019}) {
+ List<Entry<Key,Value>> sampleDataLG1 = new ArrayList<>();
+ List<Entry<Key,Value>> sampleDataLG2 = new ArrayList<>();
+
+ ConfigurationCopy sampleConf = new ConfigurationCopy(conf == null ? AccumuloConfiguration.getDefaultConfiguration() : conf);
+ sampleConf.set(Property.TABLE_SAMPLER, RowSampler.class.getName());
+ sampleConf.set(Property.TABLE_SAMPLER_OPTS + "hasher", "murmur3_32");
+ sampleConf.set(Property.TABLE_SAMPLER_OPTS + "modulus", modulus + "");
+
+ Sampler sampler = SamplerFactory.newSampler(SamplerConfigurationImpl.newSamplerConfig(sampleConf), sampleConf);
+
+ TestRFile trf = new TestRFile(sampleConf);
+
+ trf.openWriter(false, 1000);
+
+ trf.writer.startNewLocalityGroup("meta-lg", ncfs("metaA", "metaB"));
+ for (int r = 0; r < num; r++) {
+ String row = String.format("r%06d", r);
+ Key k1 = new Key(row, "metaA", "q9", 7);
+ Key k2 = new Key(row, "metaB", "q8", 7);
+ Key k3 = new Key(row, "metaB", "qA", 7);
+
+ Value v1 = new Value(("" + r).getBytes());
+ Value v2 = new Value(("" + r * 93).getBytes());
+ Value v3 = new Value(("" + r * 113).getBytes());
+
+ if (sampler.accept(k1)) {
+ sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k1, v1));
+ sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k2, v2));
+ sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k3, v3));
+ }
+
+ trf.writer.append(k1, v1);
+ trf.writer.append(k2, v2);
+ trf.writer.append(k3, v3);
+ }
+
+ trf.writer.startDefaultLocalityGroup();
+
+ for (int r = 0; r < num; r++) {
+ String row = String.format("r%06d", r);
+ Key k1 = new Key(row, "dataA", "q9", 7);
+
+ Value v1 = new Value(("" + r).getBytes());
+
+ if (sampler.accept(k1)) {
+ sampleDataLG2.add(new AbstractMap.SimpleImmutableEntry<>(k1, v1));
+ }
+
+ trf.writer.append(k1, v1);
+ }
+
+ trf.closeWriter();
+
+ Assert.assertTrue(sampleDataLG1.size() > 0);
+ Assert.assertTrue(sampleDataLG2.size() > 0);
+
+ trf.openReader(false);
+ FileSKVIterator sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf));
+
+ checkSample(sample, sampleDataLG1, ncfs("metaA", "metaB"), true);
+ checkSample(sample, sampleDataLG1, ncfs("metaA"), true);
+ checkSample(sample, sampleDataLG1, ncfs("metaB"), true);
+ checkSample(sample, sampleDataLG1, ncfs("dataA"), false);
+
+ checkSample(sample, sampleDataLG2, ncfs("metaA", "metaB"), false);
+ checkSample(sample, sampleDataLG2, ncfs("dataA"), true);
+
+ ArrayList<Entry<Key,Value>> allSampleData = new ArrayList<>();
+ allSampleData.addAll(sampleDataLG1);
+ allSampleData.addAll(sampleDataLG2);
+
+ Collections.sort(allSampleData, new Comparator<Entry<Key,Value>>() {
+ @Override
+ public int compare(Entry<Key,Value> o1, Entry<Key,Value> o2) {
+ return o1.getKey().compareTo(o2.getKey());
+ }
+ });
+
+ checkSample(sample, allSampleData, ncfs("dataA", "metaA"), true);
+ checkSample(sample, allSampleData, EMPTY_COL_FAMS, false);
+
+ trf.closeReader();
+ }
+ }
+ }
+
+ @Test
+ public void testEncSample() throws IOException {
+ conf = setAndGetAccumuloConfig(CryptoTest.CRYPTO_ON_CONF);
+ testSample();
+ testSampleLG();
+ conf = null;
+ }
+
@Test
public void testBigKeys() throws IOException {
// this test ensures that big keys do not end up index
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/file/rfile/RelativeKeyTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/IteratorUtilTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java
index 8949c92,d599674..b2de6cb
--- a/core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/system/MultiIteratorTest.java
@@@ -344,7 -344,7 +344,8 @@@ public class MultiIteratorTest extends
List<SortedKeyValueIterator<Key,Value>> skvil = new ArrayList<>(1);
skvil.add(new SortedMapIterator(tm1));
- KeyExtent extent = new KeyExtent("tablename", nr(1), nr(0));
- KeyExtent extent = new KeyExtent(new Text("tablename"), newRow(1), newRow(0));
++ KeyExtent extent = new KeyExtent("tablename", newRow(1), newRow(0));
++
MultiIterator mi = new MultiIterator(skvil, extent);
Range r1 = new Range((Text) null, (Text) null);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java
index 4ec0269,e16c912..15c424b
--- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java
@@@ -54,9 -63,22 +54,9 @@@ public class RowDeletingIteratorTest ex
public boolean isFullMajorCompaction() {
return fmc;
}
-
- @Override
- public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException {
- return null;
- }
-
- @Override
- public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {}
-
- @Override
- public Authorizations getAuthorizations() {
- return null;
- }
}
- Key nk(String row, String cf, String cq, long time) {
+ Key newKey(String row, String cf, String cq, long time) {
return new Key(new Text(row), new Text(cf), new Text(cq), time);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java
----------------------------------------------------------------------
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java
index f474972,8d8767f..221e921
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/CheckTabletMetadataTest.java
@@@ -89,17 -89,17 +89,17 @@@ public class CheckTabletMetadataTest
assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 4));
assertFail(tabletMeta, ke, new TServerInstance("127.0.0.2:9997", 5));
- assertFail(tabletMeta, new KeyExtent(new Text("1"), null, new Text("m")), tsi);
+ assertFail(tabletMeta, new KeyExtent("1", null, new Text("m")), tsi);
- assertFail(tabletMeta, new KeyExtent(new Text("1"), new Text("r"), new Text("m")), tsi);
+ assertFail(tabletMeta, new KeyExtent("1", new Text("r"), new Text("m")), tsi);
- assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN));
+ assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN));
- assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN));
+ assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN));
- assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.ServerColumnFamily.TIME_COLUMN));
+ assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.ServerColumnFamily.TIME_COLUMN));
- assertFail(tabletMeta, ke, tsi, nk("1<", TabletsSection.FutureLocationColumnFamily.NAME, "4"));
+ assertFail(tabletMeta, ke, tsi, newKey("1<", TabletsSection.FutureLocationColumnFamily.NAME, "4"));
TreeMap<Key,Value> copy = new TreeMap<>(tabletMeta);
put(copy, "1<", TabletsSection.CurrentLocationColumnFamily.NAME, "4", "127.0.0.1:9997");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
----------------------------------------------------------------------
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
index f813d42,af77150..2733efd
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
@@@ -208,11 -138,11 +208,11 @@@ public class InMemoryMapTest
mutate(imm, "r3", "foo:cq1", 3, "bar9");
mutate(imm, "r3", "foo:cq1", 3, "bara");
- MemoryIterator ski2 = imm.skvIterator();
+ MemoryIterator ski2 = imm.skvIterator(null);
ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(ski1, "r1", "foo:cq1", 3, "bar2");
- ae(ski1, "r1", "foo:cq1", 3, "bar1");
+ testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar2");
+ testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1");
assertFalse(ski1.hasTop());
ski2.seek(new Range(new Text("r3")), LocalityGroupUtil.EMPTY_CF_SET, false);
@@@ -281,9 -192,9 +281,9 @@@
mutate(imm, "r1", "foo:cq1", 3, "bar2");
mutate(imm, "r1", "foo:cq1", 3, "bar3");
- MemoryIterator ski1 = imm.skvIterator();
+ MemoryIterator ski1 = imm.skvIterator(null);
ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(ski1, "r1", "foo:cq1", 3, "bar3");
+ testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar3");
imm.delete(0);
@@@ -299,9 -210,9 +299,9 @@@
mutate(imm, "r1", "foo:cq2", 3, "bar2");
mutate(imm, "r1", "foo:cq3", 3, "bar3");
- ski1 = imm.skvIterator();
+ ski1 = imm.skvIterator(null);
ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(ski1, "r1", "foo:cq1", 3, "bar1");
+ testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1");
imm.delete(0);
@@@ -325,13 -236,13 +325,13 @@@
mutate(imm, "r1", "foo:cq5", 3, "bar5");
- SortedKeyValueIterator<Key,Value> dc = ski1.deepCopy(null);
+ SortedKeyValueIterator<Key,Value> dc = ski1.deepCopy(new SampleIE());
- ski1.seek(new Range(nk("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(ski1, "r1", "foo:cq1", 3, "bar1");
+ ski1.seek(new Range(newKey("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
+ testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1");
- dc.seek(new Range(nk("r1", "foo:cq2", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(dc, "r1", "foo:cq2", 3, "bar2");
+ dc.seek(new Range(newKey("r1", "foo:cq2", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
+ testAndCallNext(dc, "r1", "foo:cq2", 3, "bar2");
imm.delete(0);
@@@ -462,13 -373,13 +462,13 @@@
mutate(imm, "r1", "foo:cq3", 3, "bar3");
mutate(imm, "r1", "foo:cq4", 3, "bar4");
- MemoryIterator skvi1 = imm.skvIterator();
+ MemoryIterator skvi1 = imm.skvIterator(null);
- skvi1.seek(new Range(nk("r1", "foo:cq3", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(skvi1, "r1", "foo:cq3", 3, "bar3");
+ skvi1.seek(new Range(newKey("r1", "foo:cq3", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
+ testAndCallNext(skvi1, "r1", "foo:cq3", 3, "bar3");
- skvi1.seek(new Range(nk("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(skvi1, "r1", "foo:cq1", 3, "bar1");
+ skvi1.seek(new Range(newKey("r1", "foo:cq1", 3), null), LocalityGroupUtil.EMPTY_CF_SET, false);
+ testAndCallNext(skvi1, "r1", "foo:cq1", 3, "bar1");
}
@@@ -481,10 -392,10 +481,10 @@@
m.put(new Text("foo"), new Text("cq"), 3, new Value("v2".getBytes()));
imm.mutate(Collections.singletonList(m));
- MemoryIterator skvi1 = imm.skvIterator();
+ MemoryIterator skvi1 = imm.skvIterator(null);
skvi1.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
- ae(skvi1, "r1", "foo:cq", 3, "v2");
- ae(skvi1, "r1", "foo:cq", 3, "v1");
+ testAndCallNext(skvi1, "r1", "foo:cq", 3, "v2");
+ testAndCallNext(skvi1, "r1", "foo:cq", 3, "v1");
}
private static final Logger log = Logger.getLogger(InMemoryMapTest.class);
@@@ -586,259 -497,11 +586,259 @@@
// seekLocalityGroups(iter1.deepCopy(null));
}
+ @Test
+ public void testSample() throws Exception {
+
+ SamplerConfigurationImpl sampleConfig = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7"));
+ Sampler sampler = SamplerFactory.newSampler(sampleConfig, DefaultConfiguration.getInstance());
+
+ ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
+ for (Entry<String,String> entry : sampleConfig.toTablePropertiesMap().entrySet()) {
+ config1.set(entry.getKey(), entry.getValue());
+ }
+
+ ConfigurationCopy config2 = newConfig(tempFolder.newFolder().getAbsolutePath());
+ config2.set(Property.TABLE_LOCALITY_GROUP_PREFIX + "lg1", LocalityGroupUtil.encodeColumnFamilies(toTextSet("cf2")));
+ config2.set(Property.TABLE_LOCALITY_GROUPS.getKey(), "lg1");
+ for (Entry<String,String> entry : sampleConfig.toTablePropertiesMap().entrySet()) {
+ config2.set(entry.getKey(), entry.getValue());
+ }
+
+ for (ConfigurationCopy config : Arrays.asList(config1, config2)) {
+
+ InMemoryMap imm = new InMemoryMap(config);
+
+ TreeMap<Key,Value> expectedSample = new TreeMap<>();
+ TreeMap<Key,Value> expectedAll = new TreeMap<>();
+ TreeMap<Key,Value> expectedNone = new TreeMap<>();
+
+ MemoryIterator iter0 = imm.skvIterator(sampleConfig);
+
+ for (int r = 0; r < 100; r++) {
+ String row = String.format("r%06d", r);
+ mutate(imm, row, "cf1:cq1", 5, "v" + (2 * r), sampler, expectedSample, expectedAll);
+ mutate(imm, row, "cf2:cq2", 5, "v" + ((2 * r) + 1), sampler, expectedSample, expectedAll);
+ }
+
+ assertTrue(expectedSample.size() > 0);
+
+ MemoryIterator iter1 = imm.skvIterator(sampleConfig);
+ MemoryIterator iter2 = imm.skvIterator(null);
+ SortedKeyValueIterator<Key,Value> iter0dc1 = iter0.deepCopy(new SampleIE());
+ SortedKeyValueIterator<Key,Value> iter0dc2 = iter0.deepCopy(new SampleIE(sampleConfig));
+ SortedKeyValueIterator<Key,Value> iter1dc1 = iter1.deepCopy(new SampleIE());
+ SortedKeyValueIterator<Key,Value> iter1dc2 = iter1.deepCopy(new SampleIE(sampleConfig));
+ SortedKeyValueIterator<Key,Value> iter2dc1 = iter2.deepCopy(new SampleIE());
+ SortedKeyValueIterator<Key,Value> iter2dc2 = iter2.deepCopy(new SampleIE(sampleConfig));
+
+ assertEquals(expectedNone, readAll(iter0));
+ assertEquals(expectedNone, readAll(iter0dc1));
+ assertEquals(expectedNone, readAll(iter0dc2));
+ assertEquals(expectedSample, readAll(iter1));
+ assertEquals(expectedAll, readAll(iter2));
+ assertEquals(expectedAll, readAll(iter1dc1));
+ assertEquals(expectedAll, readAll(iter2dc1));
+ assertEquals(expectedSample, readAll(iter1dc2));
+ assertEquals(expectedSample, readAll(iter2dc2));
+
+ imm.delete(0);
+
+ assertEquals(expectedNone, readAll(iter0));
+ assertEquals(expectedNone, readAll(iter0dc1));
+ assertEquals(expectedNone, readAll(iter0dc2));
+ assertEquals(expectedSample, readAll(iter1));
+ assertEquals(expectedAll, readAll(iter2));
+ assertEquals(expectedAll, readAll(iter1dc1));
+ assertEquals(expectedAll, readAll(iter2dc1));
+ assertEquals(expectedSample, readAll(iter1dc2));
+ assertEquals(expectedSample, readAll(iter2dc2));
+
+ SortedKeyValueIterator<Key,Value> iter0dc3 = iter0.deepCopy(new SampleIE());
+ SortedKeyValueIterator<Key,Value> iter0dc4 = iter0.deepCopy(new SampleIE(sampleConfig));
+ SortedKeyValueIterator<Key,Value> iter1dc3 = iter1.deepCopy(new SampleIE());
+ SortedKeyValueIterator<Key,Value> iter1dc4 = iter1.deepCopy(new SampleIE(sampleConfig));
+ SortedKeyValueIterator<Key,Value> iter2dc3 = iter2.deepCopy(new SampleIE());
+ SortedKeyValueIterator<Key,Value> iter2dc4 = iter2.deepCopy(new SampleIE(sampleConfig));
+
+ assertEquals(expectedNone, readAll(iter0dc3));
+ assertEquals(expectedNone, readAll(iter0dc4));
+ assertEquals(expectedAll, readAll(iter1dc3));
+ assertEquals(expectedAll, readAll(iter2dc3));
+ assertEquals(expectedSample, readAll(iter1dc4));
+ assertEquals(expectedSample, readAll(iter2dc4));
+
+ iter1.close();
+ iter2.close();
+ }
+ }
+
+ @Test
+ public void testInterruptingSample() throws Exception {
+ runInterruptSampleTest(false, false, false);
+ runInterruptSampleTest(false, true, false);
+ runInterruptSampleTest(true, false, false);
+ runInterruptSampleTest(true, true, false);
+ runInterruptSampleTest(true, true, true);
+ }
+
+ private void runInterruptSampleTest(boolean deepCopy, boolean delete, boolean dcAfterDelete) throws Exception {
+ SamplerConfigurationImpl sampleConfig1 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "2"));
+ Sampler sampler = SamplerFactory.newSampler(sampleConfig1, DefaultConfiguration.getInstance());
+
+ ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
+ for (Entry<String,String> entry : sampleConfig1.toTablePropertiesMap().entrySet()) {
+ config1.set(entry.getKey(), entry.getValue());
+ }
+
+ InMemoryMap imm = new InMemoryMap(config1);
+
+ TreeMap<Key,Value> expectedSample = new TreeMap<>();
+ TreeMap<Key,Value> expectedAll = new TreeMap<>();
+
+ for (int r = 0; r < 1000; r++) {
+ String row = String.format("r%06d", r);
+ mutate(imm, row, "cf1:cq1", 5, "v" + (2 * r), sampler, expectedSample, expectedAll);
+ mutate(imm, row, "cf2:cq2", 5, "v" + ((2 * r) + 1), sampler, expectedSample, expectedAll);
+ }
+
+ assertTrue(expectedSample.size() > 0);
+
+ MemoryIterator miter = imm.skvIterator(sampleConfig1);
+ AtomicBoolean iFlag = new AtomicBoolean(false);
+ miter.setInterruptFlag(iFlag);
+ SortedKeyValueIterator<Key,Value> iter = miter;
+
+ if (delete && !dcAfterDelete) {
+ imm.delete(0);
+ }
+
+ if (deepCopy) {
+ iter = iter.deepCopy(new SampleIE(sampleConfig1));
+ }
+
+ if (delete && dcAfterDelete) {
+ imm.delete(0);
+ }
+
+ assertEquals(expectedSample, readAll(iter));
+ iFlag.set(true);
+ try {
+ readAll(iter);
+ Assert.fail();
+ } catch (IterationInterruptedException iie) {}
+
+ miter.close();
+ }
+
+ private void mutate(InMemoryMap imm, String row, String cols, int ts, String val, Sampler sampler, TreeMap<Key,Value> expectedSample,
+ TreeMap<Key,Value> expectedAll) {
+ mutate(imm, row, cols, ts, val);
+ Key k1 = nk(row, cols, ts);
+ if (sampler.accept(k1)) {
+ expectedSample.put(k1, new Value(val.getBytes()));
+ }
+ expectedAll.put(k1, new Value(val.getBytes()));
+ }
+
+ @Test(expected = SampleNotPresentException.class)
+ public void testDifferentSampleConfig() throws Exception {
+ SamplerConfigurationImpl sampleConfig = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7"));
+
+ ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
+ for (Entry<String,String> entry : sampleConfig.toTablePropertiesMap().entrySet()) {
+ config1.set(entry.getKey(), entry.getValue());
+ }
+
+ InMemoryMap imm = new InMemoryMap(config1);
+
+ mutate(imm, "r", "cf:cq", 5, "b");
+
+ SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
+ MemoryIterator iter = imm.skvIterator(sampleConfig2);
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ }
+
+ @Test(expected = SampleNotPresentException.class)
+ public void testNoSampleConfig() throws Exception {
+ InMemoryMap imm = newInMemoryMap(false, tempFolder.newFolder().getAbsolutePath());
+
+ mutate(imm, "r", "cf:cq", 5, "b");
+
+ SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
+ MemoryIterator iter = imm.skvIterator(sampleConfig2);
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ }
+
+ @Test
+ public void testEmptyNoSampleConfig() throws Exception {
+ InMemoryMap imm = newInMemoryMap(false, tempFolder.newFolder().getAbsolutePath());
+
+ SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
+
+ // when in mem map is empty should be able to get sample iterator with any sample config
+ MemoryIterator iter = imm.skvIterator(sampleConfig2);
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ Assert.assertFalse(iter.hasTop());
+ }
+
+ @Test
+ public void testDeferredSamplerCreation() throws Exception {
+ SamplerConfigurationImpl sampleConfig1 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "9"));
+
+ ConfigurationCopy config1 = newConfig(tempFolder.newFolder().getAbsolutePath());
+ for (Entry<String,String> entry : sampleConfig1.toTablePropertiesMap().entrySet()) {
+ config1.set(entry.getKey(), entry.getValue());
+ }
+
+ InMemoryMap imm = new InMemoryMap(config1);
+
+ // change sampler config after creating in mem map.
+ SamplerConfigurationImpl sampleConfig2 = new SamplerConfigurationImpl(RowSampler.class.getName(), ImmutableMap.of("hasher", "murmur3_32", "modulus", "7"));
+ for (Entry<String,String> entry : sampleConfig2.toTablePropertiesMap().entrySet()) {
+ config1.set(entry.getKey(), entry.getValue());
+ }
+
+ TreeMap<Key,Value> expectedSample = new TreeMap<>();
+ TreeMap<Key,Value> expectedAll = new TreeMap<>();
+ Sampler sampler = SamplerFactory.newSampler(sampleConfig2, config1);
+
+ for (int i = 0; i < 100; i++) {
+ mutate(imm, "r" + i, "cf:cq", 5, "v" + i, sampler, expectedSample, expectedAll);
+ }
+
+ MemoryIterator iter = imm.skvIterator(sampleConfig2);
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ Assert.assertEquals(expectedSample, readAll(iter));
+
+ SortedKeyValueIterator<Key,Value> dc = iter.deepCopy(new SampleIE(sampleConfig2));
+ dc.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ Assert.assertEquals(expectedSample, readAll(dc));
+
+ iter = imm.skvIterator(null);
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ Assert.assertEquals(expectedAll, readAll(iter));
+
+ iter = imm.skvIterator(sampleConfig1);
+ thrown.expect(SampleNotPresentException.class);
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+ }
+
+ private TreeMap<Key,Value> readAll(SortedKeyValueIterator<Key,Value> iter) throws IOException {
+ iter.seek(new Range(), LocalityGroupUtil.EMPTY_CF_SET, false);
+
+ TreeMap<Key,Value> actual = new TreeMap<>();
+ while (iter.hasTop()) {
+ actual.put(iter.getTopKey(), iter.getTopValue());
+ iter.next();
+ }
+ return actual;
+ }
+
private void seekLocalityGroups(SortedKeyValueIterator<Key,Value> iter1) throws IOException {
iter1.seek(new Range(), newCFSet("cf1"), true);
- ae(iter1, "r1", "cf1:x", 2, "1");
- ae(iter1, "r1", "cf1:y", 2, "2");
- ae(iter1, "r2", "cf2:x", 3, "5");
+ testAndCallNext(iter1, "r1", "cf1:x", 2, "1");
+ testAndCallNext(iter1, "r1", "cf1:y", 2, "2");
+ testAndCallNext(iter1, "r2", "cf2:x", 3, "5");
assertFalse(iter1.hasTop());
iter1.seek(new Range("r2", "r4"), newCFSet("cf1"), true);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/test/src/main/java/org/apache/accumulo/test/NativeMapPerformanceTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd70d781/test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java
index ea3ba66,0000000..fc9e8d1
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@@ -1,616 -1,0 +1,617 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Random;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.test.categories.SunnyDayTests;
+import org.apache.accumulo.tserver.NativeMap;
+import org.apache.hadoop.io.Text;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SunnyDayTests.class)
+public class NativeMapIT {
+
- private Key nk(int r) {
++ private Key newKey(int r) {
+ return new Key(new Text(String.format("r%09d", r)));
+ }
+
- private Key nk(int r, int cf, int cq, int cv, int ts, boolean deleted) {
++ private Key newKey(int r, int cf, int cq, int cv, int ts, boolean deleted) {
+ Key k = new Key(new Text(String.format("r%09d", r)), new Text(String.format("cf%09d", cf)), new Text(String.format("cq%09d", cq)), new Text(String.format(
+ "cv%09d", cv)), ts);
+
+ k.setDeleted(deleted);
+
+ return k;
+ }
+
- private Value nv(int v) {
++ private Value newValue(int v) {
+ return new Value(String.format("r%09d", v).getBytes(UTF_8));
+ }
+
+ public static File nativeMapLocation() {
+ File projectDir = new File(System.getProperty("user.dir")).getParentFile();
+ File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION);
+ return nativeMapDir;
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
+ }
+
+ private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {
+ for (int i = start; i <= end; i++) {
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
- assertEquals(nk(i), entry.getKey());
- assertEquals(nv(i + valueOffset), entry.getValue());
++ assertEquals(newKey(i), entry.getKey());
++ assertEquals(newValue(i + valueOffset), entry.getValue());
+ }
+
+ assertFalse(iter.hasNext());
+ }
+
+ private void insertAndVerify(NativeMap nm, int start, int end, int valueOffset) {
+ for (int i = start; i <= end; i++) {
- nm.put(nk(i), nv(i + valueOffset));
++ nm.put(newKey(i), newValue(i + valueOffset));
+ }
+
+ for (int i = start; i <= end; i++) {
- Value v = nm.get(nk(i));
++ Value v = nm.get(newKey(i));
+ assertNotNull(v);
- assertEquals(nv(i + valueOffset), v);
++ assertEquals(newValue(i + valueOffset), v);
+
- Iterator<Entry<Key,Value>> iter2 = nm.iterator(nk(i));
++ Iterator<Entry<Key,Value>> iter2 = nm.iterator(newKey(i));
+ assertTrue(iter2.hasNext());
+ Entry<Key,Value> entry = iter2.next();
- assertEquals(nk(i), entry.getKey());
- assertEquals(nv(i + valueOffset), entry.getValue());
++ assertEquals(newKey(i), entry.getKey());
++ assertEquals(newValue(i + valueOffset), entry.getValue());
+ }
+
- assertNull(nm.get(nk(start - 1)));
++ assertNull(nm.get(newKey(start - 1)));
+
- assertNull(nm.get(nk(end + 1)));
++ assertNull(nm.get(newKey(end + 1)));
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ verifyIterator(start, end, valueOffset, iter);
+
+ for (int i = start; i <= end; i++) {
- iter = nm.iterator(nk(i));
++ iter = nm.iterator(newKey(i));
+ verifyIterator(i, end, valueOffset, iter);
+
+ // lookup nonexistant key that falls after existing key
- iter = nm.iterator(nk(i, 1, 1, 1, 1, false));
++ iter = nm.iterator(newKey(i, 1, 1, 1, 1, false));
+ verifyIterator(i + 1, end, valueOffset, iter);
+ }
+
+ assertEquals(end - start + 1, nm.size());
+ }
+
+ private void insertAndVerifyExhaustive(NativeMap nm, int num, int run) {
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = 0; ts < num; ts++) {
- Key key = nk(i, j, k, l, ts, true);
++ Key key = newKey(i, j, k, l, ts, true);
+ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ nm.put(key, value);
+
- key = nk(i, j, k, l, ts, false);
++ key = newKey(i, j, k, l, ts, false);
+ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ nm.put(key, value);
+ }
+ }
+ }
+ }
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = num - 1; ts >= 0; ts--) {
- Key key = nk(i, j, k, l, ts, true);
++ Key key = newKey(i, j, k, l, ts, true);
+ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+
- key = nk(i, j, k, l, ts, false);
++ key = newKey(i, j, k, l, ts, false);
+ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ assertTrue(iter.hasNext());
+ entry = iter.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ assertFalse(iter.hasNext());
+
+ for (int i = 0; i < num; i++) {
+ for (int j = 0; j < num; j++) {
+ for (int k = 0; k < num; k++) {
+ for (int l = 0; l < num; l++) {
+ for (int ts = 0; ts < num; ts++) {
- Key key = nk(i, j, k, l, ts, true);
++ Key key = newKey(i, j, k, l, ts, true);
+ Value value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + true + "_" + run).getBytes(UTF_8));
+
+ assertEquals(value, nm.get(key));
+
+ Iterator<Entry<Key,Value>> iter2 = nm.iterator(key);
+ assertTrue(iter2.hasNext());
+ Entry<Key,Value> entry = iter2.next();
+ assertEquals(key, entry.getKey());
+ assertEquals(value, entry.getValue());
+
- key = nk(i, j, k, l, ts, false);
++ key = newKey(i, j, k, l, ts, false);
+ value = new Value((i + "_" + j + "_" + k + "_" + l + "_" + ts + "_" + false + "_" + run).getBytes(UTF_8));
+
+ assertEquals(value, nm.get(key));
+
+ Iterator<Entry<Key,Value>> iter3 = nm.iterator(key);
+ assertTrue(iter3.hasNext());
+ Entry<Key,Value> entry2 = iter3.next();
+ assertEquals(key, entry2.getKey());
+ assertEquals(value, entry2.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ assertEquals(num * num * num * num * num * 2, nm.size());
+ }
+
+ @Test
+ public void test1() {
+ NativeMap nm = new NativeMap();
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ assertFalse(iter.hasNext());
+ nm.delete();
+ }
+
+ @Test
+ public void test2() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+ insertAndVerify(nm, 1, 10, 1);
+ insertAndVerify(nm, 1, 10, 2);
+
+ nm.delete();
+ }
+
+ @Test
+ public void test4() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerifyExhaustive(nm, 3, 0);
+ insertAndVerifyExhaustive(nm, 3, 1);
+
+ nm.delete();
+ }
+
+ @Test
+ public void test5() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ iter.next();
+
+ nm.delete();
+
+ try {
- nm.put(nk(1), nv(1));
++ nm.put(newKey(1), newValue(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
- nm.get(nk(1));
++ nm.get(newKey(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.iterator();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
- nm.iterator(nk(1));
++ nm.iterator(newKey(1));
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ nm.size();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+
+ }
+
+ @Test
+ public void test7() {
+ NativeMap nm = new NativeMap();
+
+ insertAndVerify(nm, 1, 10, 0);
+
+ nm.delete();
+
+ try {
+ nm.delete();
+ assertTrue(false);
+ } catch (IllegalStateException e) {
+
+ }
+ }
+
+ @Test
+ public void test8() {
+ // test verifies that native map sorts keys sharing some common prefix properly
+
+ NativeMap nm = new NativeMap();
+
+ TreeMap<Key,Value> tm = new TreeMap<>();
+
+ tm.put(new Key(new Text("fo")), new Value(new byte[] {'0'}));
+ tm.put(new Key(new Text("foo")), new Value(new byte[] {'1'}));
+ tm.put(new Key(new Text("foo1")), new Value(new byte[] {'2'}));
+ tm.put(new Key(new Text("foo2")), new Value(new byte[] {'3'}));
+
+ for (Entry<Key,Value> entry : tm.entrySet()) {
+ nm.put(entry.getKey(), entry.getValue());
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ for (Entry<Key,Value> entry : tm.entrySet()) {
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry2 = iter.next();
+
+ assertEquals(entry.getKey(), entry2.getKey());
+ assertEquals(entry.getValue(), entry2.getValue());
+ }
+
+ assertFalse(iter.hasNext());
+
+ nm.delete();
+ }
+
+ @Test
+ public void test9() {
+ NativeMap nm = new NativeMap();
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (NoSuchElementException e) {
+
+ }
+
+ insertAndVerify(nm, 1, 1, 0);
+
+ iter = nm.iterator();
+ iter.next();
+
+ try {
+ iter.next();
+ assertTrue(false);
+ } catch (NoSuchElementException e) {
+
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void test10() {
+ int start = 1;
+ int end = 10000;
+
+ NativeMap nm = new NativeMap();
+ for (int i = start; i <= end; i++) {
- nm.put(nk(i), nv(i));
++ nm.put(newKey(i), newValue(i));
+ }
+
+ long mem1 = nm.getMemoryUsed();
+
+ for (int i = start; i <= end; i++) {
- nm.put(nk(i), nv(i));
++ nm.put(newKey(i), newValue(i));
+ }
+
+ long mem2 = nm.getMemoryUsed();
+
+ if (mem1 != mem2) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem2);
+ }
+
+ for (int i = start; i <= end; i++) {
- nm.put(nk(i), nv(i));
++ nm.put(newKey(i), newValue(i));
+ }
+
+ long mem3 = nm.getMemoryUsed();
+
+ if (mem1 != mem3) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem1 + " " + mem3);
+ }
+
+ byte bigrow[] = new byte[1000000];
+ byte bigvalue[] = new byte[bigrow.length];
+
+ for (int i = 0; i < bigrow.length; i++) {
+ bigrow[i] = (byte) (0xff & (i % 256));
+ bigvalue[i] = bigrow[i];
+ }
+
+ nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
+
+ long mem4 = nm.getMemoryUsed();
+
+ Value val = nm.get(new Key(new Text(bigrow)));
+ if (val == null || !val.equals(new Value(bigvalue))) {
+ throw new RuntimeException("Did not get expected big value");
+ }
+
+ nm.put(new Key(new Text(bigrow)), new Value(bigvalue));
+
+ long mem5 = nm.getMemoryUsed();
+
+ if (mem4 != mem5) {
+ throw new RuntimeException("Memory changed after inserting duplicate data " + mem4 + " " + mem5);
+ }
+
+ val = nm.get(new Key(new Text(bigrow)));
+ if (val == null || !val.equals(new Value(bigvalue))) {
+ throw new RuntimeException("Did not get expected big value");
+ }
+
+ nm.delete();
+ }
+
+ // random length random field
- private static byte[] rlrf(Random r, int maxLen) {
++ private static byte[] getRandomBytes(Random r, int maxLen) {
+ int len = r.nextInt(maxLen);
+
+ byte f[] = new byte[len];
+ r.nextBytes(f);
+
+ return f;
+ }
+
+ @Test
+ public void test11() {
+ NativeMap nm = new NativeMap();
+
+ // insert things with varying field sizes and value sizes
+
+ // generate random data
+ Random r = new Random(75);
+
+ ArrayList<Pair<Key,Value>> testData = new ArrayList<>();
+
+ for (int i = 0; i < 100000; i++) {
+
- Key k = new Key(rlrf(r, 97), rlrf(r, 13), rlrf(r, 31), rlrf(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false, false);
- Value v = new Value(rlrf(r, 511));
++ Key k = new Key(getRandomBytes(r, 97), getRandomBytes(r, 13), getRandomBytes(r, 31), getRandomBytes(r, 11), (r.nextLong() & 0x7fffffffffffffffl), false,
++ false);
++ Value v = new Value(getRandomBytes(r, 511));
+
+ testData.add(new Pair<>(k, v));
+ }
+
+ // insert unsorted data
+ for (Pair<Key,Value> pair : testData) {
+ nm.put(pair.getFirst(), pair.getSecond());
+ }
+
+ for (int i = 0; i < 2; i++) {
+
+ // sort data
+ Collections.sort(testData, new Comparator<Pair<Key,Value>>() {
+ @Override
+ public int compare(Pair<Key,Value> o1, Pair<Key,Value> o2) {
+ return o1.getFirst().compareTo(o2.getFirst());
+ }
+ });
+
+ // verify
+ Iterator<Entry<Key,Value>> iter1 = nm.iterator();
+ Iterator<Pair<Key,Value>> iter2 = testData.iterator();
+
+ while (iter1.hasNext() && iter2.hasNext()) {
+ Entry<Key,Value> e = iter1.next();
+ Pair<Key,Value> p = iter2.next();
+
+ if (!e.getKey().equals(p.getFirst()))
+ throw new RuntimeException("Keys not equal");
+
+ if (!e.getValue().equals(p.getSecond()))
+ throw new RuntimeException("Values not equal");
+ }
+
+ if (iter1.hasNext())
+ throw new RuntimeException("Not all of native map consumed");
+
+ if (iter2.hasNext())
+ throw new RuntimeException("Not all of test data consumed");
+
+ System.out.println("test 11 nm mem " + nm.getMemoryUsed());
+
+ // insert data again w/ different value
+ Collections.shuffle(testData, r);
+ // insert unsorted data
+ for (Pair<Key,Value> pair : testData) {
- pair.getSecond().set(rlrf(r, 511));
++ pair.getSecond().set(getRandomBytes(r, 511));
+ nm.put(pair.getFirst(), pair.getSecond());
+ }
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void testBinary() {
+ NativeMap nm = new NativeMap();
+
+ byte emptyBytes[] = new byte[0];
+
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ nm.put(k, v);
+ }
+ }
+
+ Iterator<Entry<Key,Value>> iter = nm.iterator();
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+
+ assertEquals(k, entry.getKey());
+ assertEquals(v, entry.getValue());
+
+ }
+ }
+
+ assertFalse(iter.hasNext());
+
+ for (int i = 0; i < 256; i++) {
+ for (int j = 0; j < 256; j++) {
+ byte row[] = new byte[] {'r', (byte) (0xff & i), (byte) (0xff & j)};
+ byte data[] = new byte[] {'v', (byte) (0xff & i), (byte) (0xff & j)};
+
+ Key k = new Key(row, emptyBytes, emptyBytes, emptyBytes, 1);
+ Value v = new Value(data);
+
+ Value v2 = nm.get(k);
+
+ assertEquals(v, v2);
+ }
+ }
+
+ nm.delete();
+ }
+
+ @Test
+ public void testEmpty() {
+ NativeMap nm = new NativeMap();
+
+ assertTrue(nm.size() == 0);
+ assertTrue(nm.getMemoryUsed() == 0);
+
+ nm.delete();
+ }
+
+ @Test
+ public void testConcurrentIter() throws IOException {
+ NativeMap nm = new NativeMap();
+
- nm.put(nk(0), nv(0));
- nm.put(nk(1), nv(1));
- nm.put(nk(3), nv(3));
++ nm.put(newKey(0), newValue(0));
++ nm.put(newKey(1), newValue(1));
++ nm.put(newKey(3), newValue(3));
+
+ SortedKeyValueIterator<Key,Value> iter = nm.skvIterator();
+
+ // modify map after iter created
- nm.put(nk(2), nv(2));
++ nm.put(newKey(2), newValue(2));
+
+ assertTrue(iter.hasTop());
- assertEquals(iter.getTopKey(), nk(0));
++ assertEquals(iter.getTopKey(), newKey(0));
+ iter.next();
+
+ assertTrue(iter.hasTop());
- assertEquals(iter.getTopKey(), nk(1));
++ assertEquals(iter.getTopKey(), newKey(1));
+ iter.next();
+
+ assertTrue(iter.hasTop());
- assertEquals(iter.getTopKey(), nk(2));
++ assertEquals(iter.getTopKey(), newKey(2));
+ iter.next();
+
+ assertTrue(iter.hasTop());
- assertEquals(iter.getTopKey(), nk(3));
++ assertEquals(iter.getTopKey(), newKey(3));
+ iter.next();
+
+ assertFalse(iter.hasTop());
+
+ nm.delete();
+ }
+
+}