You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/04/10 21:06:07 UTC
svn commit: r1466627 - in /accumulo/branches/1.4/src/server/src:
main/java/org/apache/accumulo/server/iterators/
test/java/org/apache/accumulo/server/iterators/
Author: kturner
Date: Wed Apr 10 19:06:07 2013
New Revision: 1466627
URL: http://svn.apache.org/r1466627
Log:
ACCUMULO-1044 added unit test for metdata bulk load filter
Added:
accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/iterators/
accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java
Modified: accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java?rev=1466627&r1=1466626&r2=1466627&view=diff
==============================================================================
--- accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java (original)
+++ accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilter.java Wed Apr 10 19:06:07 2013
@@ -27,6 +27,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.Arbitrator;
import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
/**
@@ -40,7 +41,7 @@ public class MetadataBulkLoadFilter exte
}
Map<Long,Status> bulkTxStatusCache;
- ZooArbitrator arbitrator;
+ Arbitrator arbitrator;
@Override
public boolean accept(Key k, Value v) {
@@ -78,6 +79,10 @@ public class MetadataBulkLoadFilter exte
}
bulkTxStatusCache = new HashMap<Long,MetadataBulkLoadFilter.Status>();
- arbitrator = new ZooArbitrator();
+ arbitrator = getArbitrator();
+ }
+
+ protected Arbitrator getArbitrator() {
+ return new ZooArbitrator();
}
}
Added: accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java?rev=1466627&view=auto
==============================================================================
--- accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java (added)
+++ accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java Wed Apr 10 19:06:07 2013
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.iterators;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.TreeMap;
+
+import junit.framework.Assert;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.apache.accumulo.core.util.ColumnFQ;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.Arbitrator;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class MetadataBulkLoadFilterTest {
+ static class TestArbitrator implements Arbitrator {
+ @Override
+ public boolean transactionAlive(String type, long tid) throws Exception {
+ return tid == 5;
+ }
+
+ @Override
+ public boolean transactionComplete(String type, long tid) throws Exception {
+ if (tid == 9)
+ throw new RuntimeException();
+ return tid != 5 && tid != 7;
+ }
+ }
+
+ static class TestMetadataBulkLoadFilter extends MetadataBulkLoadFilter {
+ @Override
+ protected Arbitrator getArbitrator() {
+ return new TestArbitrator();
+ }
+ }
+
+ private static void put(TreeMap<Key,Value> tm, String row, ColumnFQ cfq, String val) {
+ Key k = new Key(new Text(row), cfq.getColumnFamily(), cfq.getColumnQualifier());
+ tm.put(k, new Value(val.getBytes()));
+ }
+
+ private static void put(TreeMap<Key,Value> tm, String row, Text cf, String cq, String val) {
+ Key k = new Key(new Text(row), cf, new Text(cq));
+ if (val == null) {
+ k.setDeleted(true);
+ tm.put(k, new Value("".getBytes()));
+ } else
+ tm.put(k, new Value(val.getBytes()));
+ }
+
+ @Test
+ public void testBasic() throws IOException {
+ TreeMap<Key,Value> tm1 = new TreeMap<Key,Value>();
+ TreeMap<Key,Value> expected = new TreeMap<Key,Value>();
+
+ // following should not be deleted by filter
+ put(tm1, "2;m", Constants.METADATA_DIRECTORY_COLUMN, "/t1");
+ put(tm1, "2;m", Constants.METADATA_DATAFILE_COLUMN_FAMILY, "/t1/file1", "1,1");
+ put(tm1, "2;m", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t1/file1", "5");
+ put(tm1, "2;m", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t1/file3", "7");
+ put(tm1, "2;m", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t1/file4", "9");
+ put(tm1, "2<", Constants.METADATA_DIRECTORY_COLUMN, "/t2");
+ put(tm1, "2<", Constants.METADATA_DATAFILE_COLUMN_FAMILY, "/t2/file2", "1,1");
+ put(tm1, "2<", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t2/file6", "5");
+ put(tm1, "2<", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t2/file7", "7");
+ put(tm1, "2<", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t2/file8", "9");
+ put(tm1, "2<", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t2/fileC", null);
+
+ expected.putAll(tm1);
+
+ // the following should be deleted by filter
+ put(tm1, "2;m", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t1/file5", "8");
+ put(tm1, "2<", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t2/file9", "8");
+ put(tm1, "2<", Constants.METADATA_BULKFILE_COLUMN_FAMILY, "/t2/fileA", "2");
+
+ TestMetadataBulkLoadFilter iter = new TestMetadataBulkLoadFilter();
+ iter.init(new SortedMapIterator(tm1), new HashMap<String,String>(), new IteratorEnvironment() {
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName) throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean isFullMajorCompaction() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public IteratorScope getIteratorScope() {
+ return IteratorScope.majc;
+ }
+
+ @Override
+ public AccumuloConfiguration getConfig() {
+ return null;
+ }
+ });
+
+ iter.seek(new Range(), new ArrayList<ByteSequence>(), false);
+
+ TreeMap<Key,Value> actual = new TreeMap<Key,Value>();
+
+ while (iter.hasTop()) {
+ actual.put(iter.getTopKey(), iter.getTopValue());
+ iter.next();
+ }
+
+ Assert.assertEquals(expected, actual);
+ }
+}