You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2012/01/20 01:33:44 UTC
svn commit: r1233704 - in /incubator/accumulo/trunk: ./ src/core/
src/core/src/main/java/org/apache/accumulo/core/iterators/
src/core/src/main/java/org/apache/accumulo/core/iterators/user/
src/core/src/test/java/org/apache/accumulo/core/iterators/user/...
Author: kturner
Date: Fri Jan 20 00:33:44 2012
New Revision: 1233704
URL: http://svn.apache.org/viewvc?rev=1233704&view=rev
Log:
ACCUMULO-334 ACCUMULO-307 merged from 1.4
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/src/core/ (props changed)
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java
incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/VersioningIteratorTest.java
incubator/accumulo/trunk/src/server/ (props changed)
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 20 00:33:44 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043
/incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1233497
+/incubator/accumulo/branches/1.4:1201902-1233703
Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 20 00:33:44 2012
@@ -1,3 +1,3 @@
/incubator/accumulo/branches/1.3.5rc/src/core:1209938
/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1233090
+/incubator/accumulo/branches/1.4/src/core:1201902-1233703
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java Fri Jan 20 00:33:44 2012
@@ -228,6 +228,20 @@ public abstract class Combiner extends W
}
@Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ Combiner newInstance;
+ try {
+ newInstance = this.getClass().newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ newInstance.setSource(getSource().deepCopy(env));
+ newInstance.combiners = combiners;
+ newInstance.combineAllColumns = combineAllColumns;
+ return newInstance;
+ }
+
+ @Override
public IteratorOptions describeOptions() {
IteratorOptions io = new IteratorOptions("comb", "Combiners apply reduce functions to values with identical keys", null, null);
io.addNamedOption(ALL_OPTION, "set to true to apply Combiner to every column, otherwise leave blank. if true, " + COLUMNS_OPTION
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Fri Jan 20 00:33:44 2012
@@ -145,6 +145,14 @@ public abstract class TypedValueCombiner
}
}
+ @SuppressWarnings("unchecked")
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ TypedValueCombiner<V> newInstance = (TypedValueCombiner<V>) super.deepCopy(env);
+ newInstance.setEncoder(encoder);
+ return newInstance;
+ }
+
@Override
public Value reduce(Key key, Iterator<Value> iter) {
return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder))));
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/VersioningIterator.java Fri Jan 20 00:33:44 2012
@@ -29,6 +29,8 @@ public class VersioningIterator extends
public VersioningIterator() {}
public VersioningIterator(SortedKeyValueIterator<Key,Value> iterator, int maxVersions) {
- super(iterator, maxVersions);
+ super();
+ this.setSource(iterator);
+ this.maxVersions = maxVersions;
}
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/AgeOffFilter.java Fri Jan 20 00:33:44 2012
@@ -37,26 +37,6 @@ public class AgeOffFilter extends Filter
private long threshold;
private long currentTime;
- public AgeOffFilter() {}
-
- /**
- * Constructs a filter that omits entries read from a source iterator if the Key's timestamp is less than currentTime - threshold.
- *
- * @param iterator
- * The source iterator.
- *
- * @param threshold
- * Maximum age in milliseconds of data to keep.
- *
- * @param threshold
- * Current time in milliseconds.
- */
- private AgeOffFilter(SortedKeyValueIterator<Key,Value> iterator, long threshold, long currentTime) {
- setSource(iterator);
- this.threshold = threshold;
- this.currentTime = currentTime;
- }
-
/**
* Accepts entries whose timestamps are less than currentTime - threshold.
*
@@ -93,7 +73,10 @@ public class AgeOffFilter extends Filter
@Override
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new AgeOffFilter(getSource(), threshold, currentTime);
+ AgeOffFilter copy = (AgeOffFilter) super.deepCopy(env);
+ copy.currentTime = currentTime;
+ copy.threshold = threshold;
+ return copy;
}
@Override
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ColumnAgeOffFilter.java Fri Jan 20 00:33:44 2012
@@ -36,15 +36,6 @@ import org.apache.hadoop.io.Text;
* Different thresholds are set for each column.
*/
public class ColumnAgeOffFilter extends Filter {
-
- public ColumnAgeOffFilter() {}
-
- private ColumnAgeOffFilter(SortedKeyValueIterator<Key,Value> iterator, TTLSet ttls, long currentTime) {
- setSource(iterator);
- this.ttls = ttls;
- this.currentTime = currentTime;
- }
-
public static class TTLSet extends ColumnToClassMapping<Long> {
public TTLSet(Map<String,String> objectStrings) {
super();
@@ -87,7 +78,10 @@ public class ColumnAgeOffFilter extends
@Override
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new ColumnAgeOffFilter(getSource(), ttls, currentTime);
+ ColumnAgeOffFilter copy = (ColumnAgeOffFilter) super.deepCopy(env);
+ copy.currentTime = currentTime;
+ copy.ttls = ttls;
+ return copy;
}
public void overrideCurrentTime(long ts) {
@@ -123,7 +117,7 @@ public class ColumnAgeOffFilter extends
public static void addTTL(IteratorSetting is, IteratorSetting.Column column, Long ttl) {
is.addOption(ColumnSet.encodeColumns(column.getFirst(), column.getSecond()), Long.toString(ttl));
}
-
+
/**
* A convenience method for removing an age off threshold for a column.
*
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/RegExFilter.java Fri Jan 20 00:33:44 2012
@@ -37,8 +37,7 @@ public class RegExFilter extends Filter
@Override
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- RegExFilter result = new RegExFilter();
- result.setSource(getSource().deepCopy(env));
+ RegExFilter result = (RegExFilter) super.deepCopy(env);
result.rowMatcher = copyMatcher(rowMatcher);
result.colfMatcher = copyMatcher(colfMatcher);
result.colqMatcher = copyMatcher(colqMatcher);
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/TimestampFilter.java Fri Jan 20 00:33:44 2012
@@ -54,17 +54,6 @@ public class TimestampFilter extends Fil
public TimestampFilter() {}
- private TimestampFilter(SortedKeyValueIterator<Key,Value> iterator, boolean hasStart, long start, boolean startInclusive, boolean hasEnd, long end,
- boolean endInclusive) {
- setSource(iterator);
- this.start = start;
- this.startInclusive = startInclusive;
- this.hasStart = true;
- this.end = end;
- this.endInclusive = endInclusive;
- this.hasEnd = true;
- }
-
@Override
public boolean accept(Key k, Value v) {
long ts = k.getTimestamp();
@@ -112,7 +101,14 @@ public class TimestampFilter extends Fil
@Override
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- return new TimestampFilter(getSource(), hasStart, start, startInclusive, hasEnd, end, endInclusive);
+ TimestampFilter copy = (TimestampFilter) super.deepCopy(env);
+ copy.hasStart = hasStart;
+ copy.start = start;
+ copy.startInclusive = startInclusive;
+ copy.hasEnd = hasEnd;
+ copy.end = end;
+ copy.endInclusive = endInclusive;
+ return copy;
}
@Override
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/VersioningIterator.java Fri Jan 20 00:33:44 2012
@@ -37,25 +37,14 @@ public class VersioningIterator extends
private Key currentKey = new Key();
private int numVersions;
- private int maxVersions;
+ protected int maxVersions;
@Override
public VersioningIterator deepCopy(IteratorEnvironment env) {
- return new VersioningIterator(this, env);
- }
-
- private VersioningIterator(VersioningIterator other, IteratorEnvironment env) {
- setSource(other.getSource().deepCopy(env));
- maxVersions = other.maxVersions;
- }
-
- public VersioningIterator() {}
-
- public VersioningIterator(SortedKeyValueIterator<Key,Value> iterator, int maxVersions) {
- if (maxVersions < 1)
- throw new IllegalArgumentException("maxVersions for versioning iterator must be >= 1");
- this.setSource(iterator);
- this.maxVersions = maxVersions;
+ VersioningIterator copy = new VersioningIterator();
+ copy.setSource(getSource().deepCopy(env));
+ copy.maxVersions = maxVersions;
+ return copy;
}
@Override
Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java (original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Fri Jan 20 00:33:44 2012
@@ -278,6 +278,74 @@ public class CombinerTest {
}
@Test
+ public void testDeepCopy() throws IOException {
+ Encoder<Long> encoder = LongCombiner.FIXED_LEN_ENCODER;
+
+ TreeMap<Key,Value> tm1 = new TreeMap<Key,Value>();
+
+ // keys that aggregate
+ nkv(tm1, 1, 1, 1, 1, false, 2l, encoder);
+ nkv(tm1, 1, 1, 1, 2, false, 3l, encoder);
+ nkv(tm1, 1, 1, 1, 3, false, 4l, encoder);
+
+ // keys that do not aggregate
+ nkv(tm1, 2, 2, 1, 1, false, 2l, encoder);
+ nkv(tm1, 2, 2, 1, 2, false, 3l, encoder);
+
+ Combiner ai = new SummingCombiner();
+
+ IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
+ LongCombiner.setEncodingType(is, FixedLenEncoder.class.getName());
+ Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+
+ ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+
+ SortedKeyValueIterator<Key,Value> ai2 = ai.deepCopy(null);
+ SortedKeyValueIterator<Key,Value> ai3 = ai.deepCopy(null);
+ ai.seek(new Range(), EMPTY_COL_FAMS, false);
+
+ assertTrue(ai.hasTop());
+ assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
+ assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
+
+ ai.next();
+
+ assertTrue(ai.hasTop());
+ assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
+ assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
+
+ ai.next();
+
+ assertTrue(ai.hasTop());
+ assertEquals(nk(2, 2, 1, 1), ai.getTopKey());
+ assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
+
+ ai.next();
+
+ assertFalse(ai.hasTop());
+
+ // seek after key that aggregates
+ ai2.seek(nr(1, 1, 1, 2), EMPTY_COL_FAMS, false);
+
+ assertTrue(ai2.hasTop());
+ assertEquals(nk(2, 2, 1, 2), ai2.getTopKey());
+ assertEquals("3", encoder.decode(ai2.getTopValue().get()).toString());
+
+ // seek before key that aggregates
+ ai3.seek(nr(1, 1, 1, 4), EMPTY_COL_FAMS, false);
+
+ assertTrue(ai3.hasTop());
+ assertEquals(nk(1, 1, 1, 3), ai3.getTopKey());
+ assertEquals("9", encoder.decode(ai3.getTopValue().get()).toString());
+
+ ai3.next();
+
+ assertTrue(ai3.hasTop());
+ assertEquals(nk(2, 2, 1, 2), ai3.getTopKey());
+ assertEquals("3", encoder.decode(ai3.getTopValue().get()).toString());
+ }
+
+ @Test
public void test4() throws IOException {
Encoder<Long> encoder = LongCombiner.STRING_ENCODER;
Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java (original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java Fri Jan 20 00:33:44 2012
@@ -16,6 +16,9 @@
*/
package org.apache.accumulo.core.iterators.user;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -26,8 +29,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.TreeMap;
-import junit.framework.TestCase;
-
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Column;
@@ -43,13 +44,14 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.Text;
+import org.junit.Test;
-public class FilterTest extends TestCase {
+public class FilterTest {
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
private static final Map<String,String> EMPTY_OPTS = new HashMap<String,String>();
- public class SimpleFilter extends Filter {
+ public static class SimpleFilter extends Filter {
public boolean accept(Key k, Value v) {
// System.out.println(k.getRow());
if (k.getRow().toString().endsWith("0"))
@@ -58,7 +60,7 @@ public class FilterTest extends TestCase
}
}
- public class SimpleFilter2 extends Filter {
+ public static class SimpleFilter2 extends Filter {
public boolean accept(Key k, Value v) {
if (k.getColumnFamily().toString().equals("a"))
return false;
@@ -76,6 +78,7 @@ public class FilterTest extends TestCase
return size;
}
+ @Test
public void test1() throws IOException {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -110,6 +113,7 @@ public class FilterTest extends TestCase
assertTrue("size = " + size, size == 0);
}
+ @Test
public void test1neg() throws IOException {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -122,32 +126,59 @@ public class FilterTest extends TestCase
}
assertTrue(tm.size() == 1000);
- Filter filter1 = new SimpleFilter();
+ Filter filter = new SimpleFilter();
IteratorSetting is = new IteratorSetting(1, SimpleFilter.class);
Filter.setNegate(is, true);
- filter1.init(new SortedMapIterator(tm), is.getProperties(), null);
- filter1.seek(new Range(), EMPTY_COL_FAMS, false);
- int size = size(filter1);
+ filter.init(new SortedMapIterator(tm), is.getProperties(), null);
+ filter.seek(new Range(), EMPTY_COL_FAMS, false);
+ int size = size(filter);
assertTrue("size = " + size, size == 900);
- Filter fi = new SimpleFilter();
- fi.init(new SortedMapIterator(tm), is.getProperties(), null);
+ filter.init(new SortedMapIterator(tm), is.getProperties(), null);
Key k = new Key(new Text("500"));
- fi.seek(new Range(k, null), EMPTY_COL_FAMS, false);
- size = size(fi);
+ filter.seek(new Range(k, null), EMPTY_COL_FAMS, false);
+ size = size(filter);
assertTrue("size = " + size, size == 450);
- filter1 = new SimpleFilter();
- filter1.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
+ filter.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
Filter filter2 = new SimpleFilter2();
- filter2.init(filter1, is.getProperties(), null);
+ filter2.init(filter, is.getProperties(), null);
filter2.seek(new Range(), EMPTY_COL_FAMS, false);
size = size(filter2);
assertTrue("size = " + size, size == 100);
}
+ @Test
+ public void testDeepCopy() throws IOException {
+ Text colf = new Text("a");
+ Text colq = new Text("b");
+ Value dv = new Value();
+ TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
+
+ for (int i = 0; i < 1000; i++) {
+ Key k = new Key(new Text(String.format("%03d", i)), colf, colq);
+ tm.put(k, dv);
+ }
+ assertTrue(tm.size() == 1000);
+
+ SimpleFilter filter = new SimpleFilter();
+
+ IteratorSetting is = new IteratorSetting(1, SimpleFilter.class);
+ Filter.setNegate(is, true);
+
+ filter.init(new SortedMapIterator(tm), is.getProperties(), null);
+ SortedKeyValueIterator<Key,Value> copy = filter.deepCopy(null);
+ filter.seek(new Range(), EMPTY_COL_FAMS, false);
+ int size = size(filter);
+ assertTrue("size = " + size, size == 900);
+ copy.seek(new Range(), EMPTY_COL_FAMS, false);
+ size = size(copy);
+ assertTrue("size = " + size, size == 900);
+ }
+
+ @Test
public void test2() throws IOException {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -161,15 +192,21 @@ public class FilterTest extends TestCase
}
assertTrue(tm.size() == 1000);
- AgeOffFilter a = new AgeOffFilter();
+ SortedKeyValueIterator<Key,Value> a = new AgeOffFilter();
IteratorSetting is = new IteratorSetting(1, AgeOffFilter.class);
AgeOffFilter.setTTL(is, 101l);
AgeOffFilter.setCurrentTime(is, 1001l);
+ AgeOffFilter.setNegate(is, true);
a.init(new SortedMapIterator(tm), is.getProperties(), null);
+ a = a.deepCopy(null);
+ SortedKeyValueIterator<Key,Value> copy = a.deepCopy(null);
a.seek(new Range(), EMPTY_COL_FAMS, false);
- assertEquals(size(a), 100);
+ assertEquals(size(a), 900);
+ copy.seek(new Range(), EMPTY_COL_FAMS, false);
+ assertEquals(size(copy), 900);
}
+ @Test
public void test2a() throws IOException {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -199,11 +236,13 @@ public class FilterTest extends TestCase
ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("a", "b"));
a.init(new SortedMapIterator(tm), is.getProperties(), new DefaultIteratorEnvironment());
+ a = (ColumnAgeOffFilter) a.deepCopy(null);
a.overrideCurrentTime(ts);
a.seek(new Range(), EMPTY_COL_FAMS, false);
assertEquals(size(a), 902);
}
+ @Test
public void test3() throws IOException {
Value dv = new Value();
TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
@@ -248,6 +287,7 @@ public class FilterTest extends TestCase
assertTrue("size was " + size, size == 1000);
}
+ @Test
public void test4() throws IOException {
Value dv = new Value();
TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
@@ -283,6 +323,7 @@ public class FilterTest extends TestCase
return a;
}
+ @Test
public void test5() throws IOException {
Value dv = new Value();
TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
@@ -314,6 +355,7 @@ public class FilterTest extends TestCase
assertTrue(size == 3);
}
+ @Test
public void testNoVisFilter() throws IOException {
TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
Value v = new Value();
@@ -330,6 +372,7 @@ public class FilterTest extends TestCase
assertTrue("size = " + size, size == 100);
}
+ @Test
public void testTimestampFilter() throws IOException, ParseException {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -356,6 +399,7 @@ public class FilterTest extends TestCase
IteratorSetting is = new IteratorSetting(1, TimestampFilter.class);
TimestampFilter.setRange(is, "19990101010011GMT+01:00", "19990101010031GMT+01:00");
a.init(new SortedMapIterator(tm), is.getProperties(), null);
+ a = (TimestampFilter) a.deepCopy(null);
a.seek(new Range(), EMPTY_COL_FAMS, false);
assertEquals(size(a), 21);
TimestampFilter.setRange(is, baseTime + 11000, baseTime + 31000);
Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/VersioningIteratorTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/VersioningIteratorTest.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/VersioningIteratorTest.java (original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/VersioningIteratorTest.java Fri Jan 20 00:33:44 2012
@@ -16,14 +16,15 @@
*/
package org.apache.accumulo.core.iterators.user;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map.Entry;
import java.util.TreeMap;
-import junit.framework.TestCase;
-
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
@@ -33,8 +34,9 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.SortedMapIterator;
import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder;
import org.apache.hadoop.io.Text;
+import org.junit.Test;
-public class VersioningIteratorTest extends TestCase {
+public class VersioningIteratorTest {
// add test for seek function
private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
private static final Encoder<Long> encoder = LongCombiner.FIXED_LEN_ENCODER;
@@ -60,6 +62,7 @@ public class VersioningIteratorTest exte
return tmOut;
}
+ @Test
public void test1() {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -90,6 +93,7 @@ public class VersioningIteratorTest exte
}
}
+ @Test
public void test2() {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -99,7 +103,10 @@ public class VersioningIteratorTest exte
createTestData(tm, colf, colq);
try {
- VersioningIterator it = new VersioningIterator(new SortedMapIterator(tm), 3);
+ VersioningIterator it = new VersioningIterator();
+ IteratorSetting is = new IteratorSetting(1, VersioningIterator.class);
+ VersioningIterator.setMaxVersions(is, 3);
+ it.init(new SortedMapIterator(tm), is.getProperties(), null);
// after doing this seek, should only get two keys for row 1
// since we are seeking to the middle of the most recent
@@ -122,6 +129,7 @@ public class VersioningIteratorTest exte
}
}
+ @Test
public void test3() {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -170,6 +178,7 @@ public class VersioningIteratorTest exte
}
}
+ @Test
public void test4() {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -198,6 +207,7 @@ public class VersioningIteratorTest exte
}
}
+ @Test
public void test5() throws IOException {
Text colf = new Text("a");
Text colq = new Text("b");
@@ -219,4 +229,29 @@ public class VersioningIteratorTest exte
}
+ @Test
+ public void test6() throws IOException {
+ Text colf = new Text("a");
+ Text colq = new Text("b");
+
+ TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
+
+ createTestData(tm, colf, colq);
+
+ VersioningIterator it = new VersioningIterator();
+ IteratorSetting is = new IteratorSetting(1, VersioningIterator.class);
+ VersioningIterator.setMaxVersions(is, 3);
+ it.init(new SortedMapIterator(tm), is.getProperties(), null);
+ VersioningIterator it2 = it.deepCopy(null);
+
+ Key seekKey = new Key(new Text(String.format("%03d", 1)), colf, colq, 19);
+ it.seek(new Range(seekKey, false, null, true), EMPTY_COL_FAMS, false);
+ it2.seek(new Range(seekKey, false, null, true), EMPTY_COL_FAMS, false);
+
+ assertTrue(it.hasTop());
+ assertTrue(it.getTopKey().getTimestamp() == 18);
+
+ assertTrue(it2.hasTop());
+ assertTrue(it2.getTopKey().getTimestamp() == 18);
+ }
}
Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Jan 20 00:33:44 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
/incubator/accumulo/branches/1.3.5rc/src/server:1209938
-/incubator/accumulo/branches/1.4/src/server:1201902-1233497
+/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
+/incubator/accumulo/branches/1.4/src/server:1201902-1233703
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/BulkPlusOne.java Fri Jan 20 00:33:44 2012
@@ -16,8 +16,11 @@
*/
package org.apache.accumulo.server.test.randomwalk.bulk;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.Random;
+import java.util.TreeSet;
import java.util.UUID;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -34,7 +37,7 @@ import org.apache.log4j.Logger;
public class BulkPlusOne extends BulkTest {
- public static final long LOTS = 100000;
+ public static final int LOTS = 100000;
public static final int COLS = 10;
public static final int HEX_SIZE = (int) Math.ceil(Math.log(LOTS) / Math.log(16));
public static final String FMT = "r%0" + HEX_SIZE + "x";
@@ -47,25 +50,36 @@ public class BulkPlusOne extends BulkTes
FileSystem fs = (FileSystem) state.get("fs");
fs.mkdirs(fail);
int parts = rand.nextInt(10) + 1;
- long ctr = 0;
- log.debug("preparing bulk file with " + parts + " parts");
+
+ TreeSet<Integer> startRows = new TreeSet<Integer>();
+ startRows.add(0);
+ while (startRows.size() < parts)
+ startRows.add(rand.nextInt(LOTS));
+
+ List<String> printRows = new ArrayList<String>(startRows.size());
+ for (Integer row : startRows)
+ printRows.add(String.format(FMT, row));
+
+ log.debug("preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1));
String cols[] = new String[COLS];
for (int i = 0; i < cols.length; i++) {
cols[i] = String.format("%03d", i);
}
+ List<Integer> rows = new ArrayList<Integer>(startRows);
+ rows.add(LOTS);
+
for (int i = 0; i < parts; i++) {
FileSKVWriter f = FileOperations.getInstance().openWriter(dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION, fs, fs.getConf(),
AccumuloConfiguration.getDefaultConfiguration());
f.startDefaultLocalityGroup();
- int end = (int) LOTS / parts;
- if (i == parts - 1)
- end = (int) (LOTS - ctr);
- for (int j = 0; j < end; j++) {
+ int start = rows.get(i);
+ int end = rows.get(i + 1);
+ for (int j = start; j < end; j++) {
+ String row = String.format(FMT, j);
for (String col : cols) {
- f.append(new Key(String.format(FMT, ctr), "cf", col), value);
+ f.append(new Key(row, "cf", col), value);
}
- ctr++;
}
f.close();
}
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Setup.java Fri Jan 20 00:33:44 2012
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.test.randomwalk.bulk;
+import java.net.InetAddress;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -45,7 +46,9 @@ public class Setup extends Test {
@Override
public void visit(State state, Properties props) throws Exception {
Random rand = new Random();
- tableName = Integer.toHexString(Math.abs(rand.nextInt()));
+ String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+ String pid = state.getPid();
+ tableName = String.format("bulk_%s_%s_%d", hostname, pid, System.currentTimeMillis());
log.info("Starting bulk test on " + tableName);
List<PerColumnIteratorConfig> aggregators = Collections.singletonList(new PerColumnIteratorConfig(new Text("cf".getBytes()), null,
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java?rev=1233704&r1=1233703&r2=1233704&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/bulk/Verify.java Fri Jan 20 00:33:44 2012
@@ -51,6 +51,7 @@ public class Verify extends Test {
throw new Exception("Bad key at " + entry);
}
log.info("Test successful on table " + Setup.getTableName());
+ state.getConnector().tableOperations().delete(Setup.getTableName());
}
}