You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2014/03/26 18:13:25 UTC
[1/3] git commit: Fix recent warnings introduced by several tickets
Repository: accumulo
Updated Branches:
refs/heads/1.6.0-SNAPSHOT 67a11a8d9 -> 20e2b0a6a
Fix recent warnings introduced by several tickets
ACCUMULO-2005
ACCUMULO-2182
ACCUMULO-2224
ACCUMULO-2234
ACCUMULO-2539
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4ee0f649
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4ee0f649
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4ee0f649
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 4ee0f6494ea0d2265d930bce962ebcc3c7dbfe5c
Parents: 88761e0
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Mar 26 12:45:35 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Mar 26 12:47:05 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/client/impl/OfflineScanner.java | 2 --
.../main/java/org/apache/accumulo/core/iterators/Combiner.java | 4 ++--
.../main/java/org/apache/accumulo/core/util/AddressUtil.java | 5 ++---
.../java/org/apache/accumulo/core/util/AddressUtilTest.java | 2 +-
.../org/apache/accumulo/server/test/functional/RunTests.java | 2 +-
.../java/org/apache/accumulo/server/test/randomwalk/State.java | 1 -
6 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ee0f649/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java b/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index 244bfef..cc83230 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@ -22,12 +22,10 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.RowIterator;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ee0f649/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java b/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
index 083e93e..e8e0d8c 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
@@ -38,8 +38,8 @@ import org.apache.log4j.Logger;
/**
* A SortedKeyValueIterator that combines the Values for different versions (timestamp) of a Key within a row into a single Value. Combiner will replace one or
- * more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method. An {@link IteratorSetting.Column}
- * which only specifies a column family will combine all Keys in that column family individually. Similarly, a {@link IteratorSetting.Column} which specifies a
+ * more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method. An {@link Column}
+ * which only specifies a column family will combine all Keys in that column family individually. Similarly, a {@link Column} which specifies a
* column family and column qualifier will combine all Keys in column family and qualifier individually. Combination is only ever performed on multiple versions
* and not across column qualifiers or column visibilities.
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ee0f649/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
index 96c2e18..05b7198 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
@@ -16,14 +16,13 @@
*/
package org.apache.accumulo.core.util;
-import java.net.InetAddress; // workaround to enable @see/@link hyperlink
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.security.Security;
import org.apache.hadoop.io.Text;
import org.apache.thrift.transport.TSocket;
-
import org.apache.log4j.Logger;
public class AddressUtil {
@@ -60,7 +59,7 @@ public class AddressUtil {
*
* @param originalException the host lookup that is the source of needing this lookup. maybe be null.
* @return positive integer number of seconds
- * @see java.net.InetAddress
+ * @see InetAddress
* @throws IllegalArgumentException if dns failures are cached forever
*/
static public int getAddressCacheNegativeTtl(UnknownHostException originalException) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ee0f649/src/core/src/test/java/org/apache/accumulo/core/util/AddressUtilTest.java
----------------------------------------------------------------------
diff --git a/src/core/src/test/java/org/apache/accumulo/core/util/AddressUtilTest.java b/src/core/src/test/java/org/apache/accumulo/core/util/AddressUtilTest.java
index 6268fe9..a8f1ded 100644
--- a/src/core/src/test/java/org/apache/accumulo/core/util/AddressUtilTest.java
+++ b/src/core/src/test/java/org/apache/accumulo/core/util/AddressUtilTest.java
@@ -113,7 +113,7 @@ public class AddressUtilTest extends TestCase {
}
try {
log.info("AddressUtil is (hopefully) going to spit out an error about DNS lookups. you can ignore it.");
- int result = AddressUtil.getAddressCacheNegativeTtl(null);
+ AddressUtil.getAddressCacheNegativeTtl(null);
fail("The JVM Security settings cache DNS failures forever, this should cause an exception.");
} catch(IllegalArgumentException exception) {
assertTrue(true);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ee0f649/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java b/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
index 31e3198..7a7e7d3 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
@@ -134,7 +134,7 @@ public class RunTests extends Configured implements Tool {
}
@Override
- protected void setup(Mapper.Context context) throws IOException, InterruptedException {
+ protected void setup(Mapper<LongWritable,Text,Text,Text>.Context context) throws IOException, InterruptedException {
mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, timeoutFactor));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ee0f649/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/State.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/State.java b/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/State.java
index 5a53340..a94436a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/State.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/State.java
@@ -24,7 +24,6 @@ import java.util.Properties;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
-import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.log4j.Logger;
[3/3] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/20e2b0a6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/20e2b0a6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/20e2b0a6
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 20e2b0a6a6ca56e3e91fbf9ac1c1108b6b98dc9d
Parents: 67a11a8 d77cba3
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Mar 26 13:06:34 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Mar 26 13:06:34 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/iterators/Combiner.java | 21 ++++++--------------
1 file changed, 6 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/20e2b0a6/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
index 58071cf,e8921e5..ceb4411
--- a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
@@@ -80,21 -80,21 +80,11 @@@ public abstract class Combiner extends
return source.hasTop() && !source.getTopKey().isDeleted() && topKey.equals(source.getTopKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS);
}
-- /**
-- * @return <tt>true</tt> if there is another Value
-- *
-- * @see java.util.Iterator#hasNext()
-- */
@Override
public boolean hasNext() {
return hasNext;
}
-- /**
-- * @return the next Value
-- *
-- * @see java.util.Iterator#next()
-- */
@Override
public Value next() {
if (!hasNext)
@@@ -110,9 -110,9 +100,10 @@@
}
/**
-- * unsupported
++ * This method is unsupported in this iterator.
*
-- * @see java.util.Iterator#remove()
++ * @throws UnsupportedOperationException
++ * when called
*/
@Override
public void remove() {
@@@ -272,7 -272,7 +263,7 @@@
}
if (!options.containsKey(COLUMNS_OPTION))
throw new IllegalArgumentException("options must include " + ALL_OPTION + " or " + COLUMNS_OPTION);
--
++
String encodedColumns = options.get(COLUMNS_OPTION);
if (encodedColumns.length() == 0)
throw new IllegalArgumentException("empty columns specified in option " + COLUMNS_OPTION);
[2/3] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by ct...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Conflicts:
fate/src/main/java/org/apache/accumulo/fate/util/AddressUtil.java
test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
Include warning from ACCUMULO-2433
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d77cba39
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d77cba39
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d77cba39
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: d77cba39fc032b327c448931e3841e4aa34c90f2
Parents: 5413823 4ee0f64
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Mar 26 12:56:08 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Mar 26 12:56:08 2014 -0400
----------------------------------------------------------------------
.../main/java/org/apache/accumulo/core/iterators/Combiner.java | 4 ++--
.../main/java/org/apache/accumulo/test/functional/RunTests.java | 2 +-
test/src/test/java/org/apache/accumulo/test/ShellServerTest.java | 2 --
3 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d77cba39/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
index 58071cf,0000000..e8921e5
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
@@@ -1,323 -1,0 +1,323 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.IteratorSetting.Column;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.conf.ColumnSet;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+/**
+ * A SortedKeyValueIterator that combines the Values for different versions (timestamp) of a Key within a row into a single Value. Combiner will replace one or
- * more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method. An {@link IteratorSetting.Column}
- * which only specifies a column family will combine all Keys in that column family individually. Similarly, a {@link IteratorSetting.Column} which specifies a
++ * more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method. An {@link Column}
++ * which only specifies a column family will combine all Keys in that column family individually. Similarly, a {@link Column} which specifies a
+ * column family and column qualifier will combine all Keys in column family and qualifier individually. Combination is only ever performed on multiple versions
+ * and not across column qualifiers or column visibilities.
+ *
+ * Implementations must provide a reduce method: {@code public Value reduce(Key key, Iterator<Value> iter)}.
+ *
+ * This reduce method will be passed the most recent Key and an iterator over the Values for all non-deleted versions of that Key. A combiner will not combine
+ * keys that differ by more than the timestamp.
+ *
+ * This class and its implementations do not automatically filter out unwanted columns from those being combined, thus it is generally recommended to use a
+ * {@link Combiner} implementation with the {@link ScannerBase#fetchColumnFamily(Text)} or {@link ScannerBase#fetchColumn(Text, Text)} methods.
+ */
+public abstract class Combiner extends WrappingIterator implements OptionDescriber {
+ static final Logger log = Logger.getLogger(Combiner.class);
+ protected static final String COLUMNS_OPTION = "columns";
+ protected static final String ALL_OPTION = "all";
+
+ /**
+ * A Java Iterator that iterates over the Values for a given Key from a source SortedKeyValueIterator.
+ */
+ public static class ValueIterator implements Iterator<Value> {
+ Key topKey;
+ SortedKeyValueIterator<Key,Value> source;
+ boolean hasNext;
+
+ /**
+ * Constructs an iterator over Values whose Keys are versions of the current topKey of the source SortedKeyValueIterator.
+ *
+ * @param source
+ * The SortedKeyValueIterator<Key,Value> from which to read data.
+ */
+ public ValueIterator(SortedKeyValueIterator<Key,Value> source) {
+ this.source = source;
+ topKey = new Key(source.getTopKey());
+ hasNext = _hasNext();
+ }
+
+ private boolean _hasNext() {
+ return source.hasTop() && !source.getTopKey().isDeleted() && topKey.equals(source.getTopKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS);
+ }
+
+ /**
+ * @return <tt>true</tt> if there is another Value
+ *
+ * @see java.util.Iterator#hasNext()
+ */
+ @Override
+ public boolean hasNext() {
+ return hasNext;
+ }
+
+ /**
+ * @return the next Value
+ *
+ * @see java.util.Iterator#next()
+ */
+ @Override
+ public Value next() {
+ if (!hasNext)
+ throw new NoSuchElementException();
+ Value topValue = new Value(source.getTopValue());
+ try {
+ source.next();
+ hasNext = _hasNext();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return topValue;
+ }
+
+ /**
+ * unsupported
+ *
+ * @see java.util.Iterator#remove()
+ */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ Key topKey;
+ Value topValue;
+
+ @Override
+ public Key getTopKey() {
+ if (topKey == null)
+ return super.getTopKey();
+ return topKey;
+ }
+
+ @Override
+ public Value getTopValue() {
+ if (topKey == null)
+ return super.getTopValue();
+ return topValue;
+ }
+
+ @Override
+ public boolean hasTop() {
+ return topKey != null || super.hasTop();
+ }
+
+ @Override
+ public void next() throws IOException {
+ if (topKey != null) {
+ topKey = null;
+ topValue = null;
+ } else {
+ super.next();
+ }
+
+ findTop();
+ }
+
+ private Key workKey = new Key();
+
+ /**
+ * Sets the topKey and topValue based on the top key of the source. If the column of the source top key is in the set of combiners, topKey will be the top key
+ * of the source and topValue will be the result of the reduce method. Otherwise, topKey and topValue will be unchanged. (They are always set to null before
+ * this method is called.)
+ */
+ private void findTop() throws IOException {
+ // check if aggregation is needed
+ if (super.hasTop()) {
+ workKey.set(super.getTopKey());
+ if (combineAllColumns || combiners.contains(workKey)) {
+ if (workKey.isDeleted())
+ return;
+ topKey = workKey;
+ Iterator<Value> viter = new ValueIterator(getSource());
+ topValue = reduce(topKey, viter);
+ while (viter.hasNext())
+ viter.next();
+ }
+ }
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+ // do not want to seek to the middle of a value that should be combined...
+
+ Range seekRange = IteratorUtil.maximizeStartKeyTimeStamp(range);
+
+ super.seek(seekRange, columnFamilies, inclusive);
+ findTop();
+
+ if (range.getStartKey() != null) {
+ while (hasTop() && getTopKey().equals(range.getStartKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS)
+ && getTopKey().getTimestamp() > range.getStartKey().getTimestamp()) {
+ // the value has a more recent time stamp, so pass it up
+ // log.debug("skipping "+getTopKey());
+ next();
+ }
+
+ while (hasTop() && range.beforeStartKey(getTopKey())) {
+ next();
+ }
+ }
+ }
+
+ /**
+ * Reduces a list of Values into a single Value.
+ *
+ * @param key
+ * The most recent version of the Key being reduced.
+ *
+ * @param iter
+ * An iterator over the Values for different versions of the key.
+ *
+ * @return The combined Value.
+ */
+ public abstract Value reduce(Key key, Iterator<Value> iter);
+
+ private ColumnSet combiners;
+ private boolean combineAllColumns;
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+ super.init(source, options, env);
+
+ combineAllColumns = false;
+ if (options.containsKey(ALL_OPTION)) {
+ combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+ if (combineAllColumns)
+ return;
+ }
+ if (!options.containsKey(COLUMNS_OPTION))
+ throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option");
+
+ String encodedColumns = options.get(COLUMNS_OPTION);
+ if (encodedColumns.length() == 0)
+ throw new IllegalArgumentException("The " + COLUMNS_OPTION + " must not be empty");
+
+ combiners = new ColumnSet(Arrays.asList(encodedColumns.split(",")));
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ Combiner newInstance;
+ try {
+ newInstance = this.getClass().newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ newInstance.setSource(getSource().deepCopy(env));
+ newInstance.combiners = combiners;
+ newInstance.combineAllColumns = combineAllColumns;
+ return newInstance;
+ }
+
+ @Override
+ public IteratorOptions describeOptions() {
+ IteratorOptions io = new IteratorOptions("comb", "Combiners apply reduce functions to multiple versions of values with otherwise equal keys", null, null);
+ io.addNamedOption(ALL_OPTION, "set to true to apply Combiner to every column, otherwise leave blank. if true, " + COLUMNS_OPTION
+ + " option will be ignored.");
+ io.addNamedOption(COLUMNS_OPTION, "<col fam>[:<col qual>]{,<col fam>[:<col qual>]} escape non-alphanum chars using %<hex>.");
+ return io;
+ }
+
+ @Override
+ public boolean validateOptions(Map<String,String> options) {
+ if (options.containsKey(ALL_OPTION)) {
+ try {
+ combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+ } catch (Exception e) {
+ throw new IllegalArgumentException("bad boolean " + ALL_OPTION + ":" + options.get(ALL_OPTION));
+ }
+ if (combineAllColumns)
+ return true;
+ }
+ if (!options.containsKey(COLUMNS_OPTION))
+ throw new IllegalArgumentException("options must include " + ALL_OPTION + " or " + COLUMNS_OPTION);
+
+ String encodedColumns = options.get(COLUMNS_OPTION);
+ if (encodedColumns.length() == 0)
+ throw new IllegalArgumentException("empty columns specified in option " + COLUMNS_OPTION);
+
+ for (String columns : encodedColumns.split(",")) {
+ if (!ColumnSet.isValidEncoding(columns))
+ throw new IllegalArgumentException("invalid column encoding " + encodedColumns);
+ }
+
+ return true;
+ }
+
+ /**
+ * A convenience method to set which columns a combiner should be applied to. For each column specified, all versions of a Key which match that @{link
+ * IteratorSetting.Column} will be combined individually in each row. This method is likely to be used in conjunction with
+ * {@link ScannerBase#fetchColumnFamily(Text)} or {@link ScannerBase#fetchColumn(Text,Text)}.
+ *
+ * @param is
+ * iterator settings object to configure
+ * @param columns
+ * a list of columns to encode as the value for the combiner column configuration
+ */
+
+ public static void setColumns(IteratorSetting is, List<IteratorSetting.Column> columns) {
+ String sep = "";
+ StringBuilder sb = new StringBuilder();
+
+ for (Column col : columns) {
+ sb.append(sep);
+ sep = ",";
+ sb.append(ColumnSet.encodeColumns(col.getFirst(), col.getSecond()));
+ }
+
+ is.addOption(COLUMNS_OPTION, sb.toString());
+ }
+
+ /**
+ * A convenience method to set the "all columns" option on a Combiner. This will combine all columns individually within each row.
+ *
+ * @param is
+ * iterator settings object to configure
+ * @param combineAllColumns
+ * if true, the columns option is ignored and the Combiner will be applied to all columns
+ */
+ public static void setCombineAllColumns(IteratorSetting is, boolean combineAllColumns) {
+ is.addOption(ALL_OPTION, Boolean.toString(combineAllColumns));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d77cba39/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
index b70afd1,0000000..f6ebe87
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
@@@ -1,216 -1,0 +1,216 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Runs the functional tests via map-reduce.
+ *
+ * First, be sure everything is compiled.
+ *
+ * Second, get a list of the tests you want to run:
+ *
+ * <pre>
+ * $ python test/system/auto/run.py -l > tests
+ * </pre>
+ *
+ * Put the list of tests into HDFS:
+ *
+ * <pre>
+ * $ hadoop fs -put tests /user/hadoop/tests
+ * </pre>
+ *
+ * Run the map-reduce job:
+ *
+ * <pre>
+ * $ ./bin/accumulo accumulo.test.functional.RunTests --tests /user/hadoop/tests --output /user/hadoop/results
+ * </pre>
+ *
+ * Note that you will need to have some configuration in conf/accumulo-site.xml (to locate zookeeper). The map-reduce jobs will not use your local accumulo
+ * instance.
+ *
+ */
+public class RunTests extends Configured implements Tool {
+
+ static final public String JOB_NAME = "Functional Test Runner";
+ private static final Logger log = Logger.getLogger(RunTests.class);
+
+ private Job job = null;
+
+ private static final int DEFAULT_TIMEOUT_FACTOR = 1;
+
+ static class Opts extends Help {
+ @Parameter(names="--tests", description="newline separated list of tests to run", required=true)
+ String testFile;
+ @Parameter(names="--output", description="destination for the results of tests in HDFS", required=true)
+ String outputPath;
+ @Parameter(names="--timeoutFactor", description="Optional scaling factor for timeout for both mapred.task.timeout and -f flag on run.py", required=false)
+ Integer intTimeoutFactor = DEFAULT_TIMEOUT_FACTOR;
+ }
+
+ static final String TIMEOUT_FACTOR = RunTests.class.getName() + ".timeoutFactor";
+
+ static public class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
+
+ private static final String REDUCER_RESULT_START = "::::: ";
+ private static final int RRS_LEN = REDUCER_RESULT_START.length();
+ private Text result = new Text();
+ String mapperTimeoutFactor = null;
+
+ private static enum Outcome {
+ SUCCESS, FAILURE, ERROR, UNEXPECTED_SUCCESS, EXPECTED_FAILURE
+ }
+ private static final Map<Character, Outcome> OUTCOME_COUNTERS;
+ static {
+ OUTCOME_COUNTERS = new java.util.HashMap<Character, Outcome>();
+ OUTCOME_COUNTERS.put('S', Outcome.SUCCESS);
+ OUTCOME_COUNTERS.put('F', Outcome.FAILURE);
+ OUTCOME_COUNTERS.put('E', Outcome.ERROR);
+ OUTCOME_COUNTERS.put('T', Outcome.UNEXPECTED_SUCCESS);
+ OUTCOME_COUNTERS.put('G', Outcome.EXPECTED_FAILURE);
+ }
+
+ @Override
+ protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
+ List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-f", mapperTimeoutFactor, "-t", value.toString());
+ log.info("Running test " + cmd);
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ pb.directory(new File(context.getConfiguration().get("accumulo.home")));
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+ p.getOutputStream().close();
+ InputStream out = p.getInputStream();
+ InputStreamReader outr = new InputStreamReader(out, Constants.UTF8);
+ BufferedReader br = new BufferedReader(outr);
+ String line;
+ try {
+ while ((line = br.readLine()) != null) {
+ log.info("More: " + line);
+ if (line.startsWith(REDUCER_RESULT_START)) {
+ String resultLine = line.substring(RRS_LEN);
+ if (resultLine.length() > 0) {
+ Outcome outcome = OUTCOME_COUNTERS.get(resultLine.charAt(0));
+ if (outcome != null) {
+ context.getCounter(outcome).increment(1);
+ }
+ }
+ String taskAttemptId = context.getTaskAttemptID().toString();
+ result.set(taskAttemptId + " " + resultLine);
+ context.write(value, result);
+ }
+ }
+ } catch (Exception ex) {
+ log.error(ex);
+ context.progress();
+ }
+
+ p.waitFor();
+ }
+
+ @Override
- protected void setup(Mapper.Context context) throws IOException, InterruptedException {
++ protected void setup(Mapper<LongWritable,Text,Text,Text>.Context context) throws IOException, InterruptedException {
+ mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, DEFAULT_TIMEOUT_FACTOR));
+ }
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ job = new Job(getConf(), JOB_NAME);
+ job.setJarByClass(this.getClass());
+ Opts opts = new Opts();
+ opts.parseArgs(RunTests.class.getName(), args);
+
+ // this is like 1-2 tests per mapper
+ Configuration conf = job.getConfiguration();
+ conf.setInt("mapred.max.split.size", 40);
+ conf.set("accumulo.home", System.getenv("ACCUMULO_HOME"));
+
+ // Taking third argument as scaling factor to setting mapred.task.timeout
+ // and TIMEOUT_FACTOR
+ conf.setInt("mapred.task.timeout", opts.intTimeoutFactor * 8 * 60 * 1000);
+ conf.setInt(TIMEOUT_FACTOR, opts.intTimeoutFactor);
+ conf.setBoolean("mapred.map.tasks.speculative.execution", false);
+
+ // set input
+ job.setInputFormatClass(TextInputFormat.class);
+ TextInputFormat.setInputPaths(job, new Path(opts.testFile));
+
+ // set output
+ job.setOutputFormatClass(TextOutputFormat.class);
+ FileSystem fs = FileSystem.get(conf);
+ Path destination = new Path(opts.outputPath);
+ if (fs.exists(destination)) {
+ log.info("Deleting existing output directory " + opts.outputPath);
+ fs.delete(destination, true);
+ }
+ TextOutputFormat.setOutputPath(job, destination);
+
+ // configure default reducer: put the results into one file
+ job.setNumReduceTasks(1);
+
+ // set mapper
+ job.setMapperClass(TestMapper.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ // don't do anything with the results (yet) a summary would be nice
+ job.setNumReduceTasks(0);
+
+ // submit the job
+ log.info("Starting tests");
+ return 0;
+ }
+
+ /**
+ * @param args
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ RunTests tests = new RunTests();
+ ToolRunner.run(new Configuration(), tests, args);
+ tests.job.waitForCompletion(true);
+ if (!tests.job.isSuccessful())
+ System.exit(1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d77cba39/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index caa3dd1,0000000..9b2ab33
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@@ -1,977 -1,0 +1,975 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import jline.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.server.trace.TraceServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
- import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+public class ShellServerTest {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static abstract class ErrorMessageCallback {
+ public abstract String getErrorMessage();
+ }
+
+ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
+ private static final String empty = "";
+ public String getErrorMessage() {
+ return empty;
+ }
+ }
+
- private static final Logger log = Logger.getLogger(ShellServerTest.class);
+ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
+
+ private static String secret = "superSecret";
+ public static TemporaryFolder folder = new TemporaryFolder();
+ public static MiniAccumuloCluster cluster;
+ public TestOutputStream output;
+ public Shell shell;
+ private static Process traceProcess;
+
+ @Rule
+ public TestName name = new TestName();
+
+ String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit("", true, callback);
+ else
+ assertBadExit("", true, callback);
+ return result;
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit(expectString, stringPresent, callback);
+ else
+ assertBadExit(expectString, stringPresent, callback);
+ return result;
+ }
+
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.debug(output.get());
+ if (0 != shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(errorMsg, 0, shell.getExitCode());
+ }
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.debug(output.get());
+ if (0 == shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertTrue(errorMsg, shell.getExitCode() > 0);
+ }
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ folder.create();
+ MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
+ cluster = new MiniAccumuloCluster(cfg);
+ cluster.start();
+
+ System.setProperty("HOME", folder.getRoot().getAbsolutePath());
+
+ // use reflection to call this method so it does not need to be made public
+ Method method = cluster.getClass().getDeclaredMethod("exec", Class.class, String[].class);
+ method.setAccessible(true);
+ traceProcess = (Process) method.invoke(cluster, TraceServer.class, new String[0]);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @Before
+ public void setupShell() throws Exception {
+ // start the shell
+ output = new TestOutputStream();
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", secret, "-z", cluster.getInstanceName(), cluster.getZooKeepers());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ cluster.stop();
+ traceProcess.destroy();
+ folder.delete();
+ }
+
+ @Test(timeout = 60000)
+ public void exporttableImporttable() throws Exception {
+ final String table = name.getMethodName(), table2 = table + "2";
+
+ // exporttable / importtable
+ exec("createtable " + table + " -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t " + table + " -s table.split.threshold=345M", true);
+ exec("offline " + table, true);
+ String export = folder.newFolder().toString();
+ exec("exporttable -t " + table + " " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = folder.newFolder().toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable " + table2 + " " + import_, true);
+ exec("config -t " + table2 + " -np", true, "345M", true);
+ exec("getsplits -t " + table2, true, "row5", true);
+ exec("constraint --list -t " + table2, true, "VisibilityConstraint=1", true);
+ exec("onlinetable " + table, true);
+ exec("deletetable -f " + table, true);
+ exec("deletetable -f " + table2, true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
+ @Test(timeout = 45000)
+ public void setscaniterDeletescaniter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setscaniter, deletescaniter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ shell.getReader().setInput(new ByteArrayInputStream("true\n\n\nSTRING\n".getBytes()));
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = folder.newFile();
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
+ @Test(timeout = 45000)
+ public void egrep() throws Exception {
+ final String table = name.getMethodName();
+
+ // egrep
+ exec("createtable " + table);
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void du() throws Exception {
+ final String table = name.getMethodName();
+
+ // create and delete a table so we get out of a table context in the shell
+ exec("notable", true);
+
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
+
+ output.clear();
+ exec("createtable " + table);
+ make10();
+ exec("flush -t " + table + " -w");
+ exec("du " + table, true, " [" + table + "]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[" + table + "\\]\\n"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
+ @Test(timeout = 45000)
+ public void user() throws Exception {
+ final String table = name.getMethodName();
+
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ shell.getReader().setInput(new ByteArrayInputStream("secret\nsecret\n".getBytes()));
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (!METADATA): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t !METADATA Table.WRITE", true);
+ exec("grant -u root -t !METADATA Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t !METADATA foo", false);
+ shell.getReader().setInput(new ByteArrayInputStream("secret\nsecret\n".getBytes()));
+ exec("user xyzzy", true);
+ exec("createtable " + table, true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f " + table, true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ shell.getReader().setInput(new ByteArrayInputStream((secret + "\n" + secret + "\n").getBytes()));
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t !METADATA Table.WRITE", true);
+ exec("revoke -u xyzzy -t !METADATA Table.GOOFY", false);
+ exec("revoke -u xyzzy -t !METADATA foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
+ @Test(timeout = 45000)
+ public void iter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ shell.getReader().setInput(new ByteArrayInputStream("true\n\n\nSTRING\n".getBytes()));
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ shell.getReader().setInput(new ByteArrayInputStream("true\n\n\nSTRING\n".getBytes()));
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f " + table);
+ // list iter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ shell.getReader().setInput(new ByteArrayInputStream("true\n\n\nSTRING\n".getBytes()));
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ shell.getReader().setInput(new ByteArrayInputStream("true\n\n\nSTRING\n".getBytes()));
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void notable() throws Exception {
+ final String table = name.getMethodName();
+
+ // notable
+ exec("createtable " + table, true);
+ exec("scan", true, " " + table + ">", true);
+ assertTrue(output.get().contains(" " + table + ">"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" " + table + ">"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue("Diff was actually " + diff, diff >= 200);
+ assertTrue("Diff was actually " + diff, diff < 600);
+ }
+
+ @Test(timeout = 45000)
+ public void addauths() throws Exception {
+ final String table = name.getMethodName();
+ // addauths
+ exec("createtable " + table + " -evc");
+ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
+ public String getErrorMessage() {
+ try {
+ Connector c = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getConnector("root", new PasswordToken(secret));
+ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+ });
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
+ @Test(timeout = 45000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ }
+
+ @Test(timeout = 45000)
+ public void clonetable() throws Exception {
+ final String table = name.getMethodName(), clone = table + "_clone";
+
+ // clonetable
+ exec("createtable " + table + " -evc");
+ exec("config -t " + table + " -s table.split.threshold=123M", true);
+ exec("addsplits -t " + table + " a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable " + table + " " + clone);
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t " + clone, true, "VisibilityConstraint=1", true);
+ exec("config -t " + clone + " -np", true, "123M", true);
+ exec("getsplits -t " + clone, true, "a\nb\nc\n");
+ exec("deletetable -f " + table);
+ exec("deletetable -f " + clone);
+ }
+
+ @Test(timeout = 45000)
+ public void splitMerge() throws Exception {
+ final String table = name.getMethodName();
+
+ // compact
+ exec("createtable " + table);
+
+ String tableId = getTableId(table);
+
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles(tableId);
+ // merge two files into one
+ exec("compact -t " + table + " -w");
+ assertTrue(countFiles(tableId) < oldCount);
+ exec("addsplits -t " + table + " f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
+ List<String> oldFiles = getFiles(tableId);
+
+ // at this point there are 4 files in the default tablet
+ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
+
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertEquals(2, countFiles(tableId));
+ exec("compact -w");
+ assertEquals(2, countFiles(tableId));
+ exec("merge --all -t " + table);
+ exec("compact -w");
+ assertEquals(1, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void constraint() throws Exception {
+ final String table = name.getMethodName();
+
+ // constraint
+ exec("constraint -l -t !METADATA", true, "MetadataConstraints=1", true);
+ exec("createtable " + table + " -evc");
+
+ // Make sure the table is fully propagated through zoocache
+ getTableId(table);
+
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=1", true);
+ Thread.sleep(250);
+ exec("constraint -t " + table + " -d 1", true, "Removed constraint 1 from table c");
+ Thread.sleep(250);
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=1", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deletemany() throws Exception {
+ final String table = name.getMethodName();
+
+ // deletemany
+ exec("createtable " + table);
+ make10();
+ assertEquals(10, countkeys(table));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys(table));
+ exec("scan -t " + table + " -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys(table));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deleterows() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ final String tableId = getTableId(table);
+
+ // deleterows
+ int base = countFiles(tableId);
+ assertEquals(0, base);
+
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t " + table);
+ List<String> files = getFiles(tableId);
+ assertEquals("Found the following files: " + files, 3, files.size());
+ exec("deleterows -t " + table + " -b row5 -e row7", true);
+ assertEquals(2, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void groups() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("setgroups -t " + table + " alpha=a,b,c num=3,2,1");
+ exec("getgroups -t " + table, true, "alpha=a,b,c", true);
+ exec("getgroups -t " + table, true, "num=1,2,3", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void grep() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ shell.getReader().setInput(new ByteArrayInputStream("\n\n".getBytes()));
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
+
+ // @Test(timeout = 45000)
+ public void history() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("history -c", true);
+ exec("createtable " + table);
+ exec("deletetable -f " + table);
+ exec("history", true, table, true);
+ exec("history", true, "history", true);
+ }
+
+ @Test(timeout = 45000)
+ public void importDirectory() throws Exception {
+ final String table = name.getMethodName();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = folder.newFolder("import");
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = folder.newFolder("errors");
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ exec("createtable " + table, true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
+ @Test(timeout = 45000)
+ public void interpreter() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+
+ // Need to allow time for this to propagate through zoocache/zookeeper
+ Thread.sleep(3000);
+
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void listcompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.minc.slow.opt.sleepTime=1000", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t " + table, true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void maxrow() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void merge() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f " + table);
+ exec("getsplits -t !METADATA", true);
+ assertEquals(3, output.get().split("\n").length);
+ exec("merge --all -t !METADATA");
+ exec("getsplits -t !METADATA", true);
+ assertEquals(2, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void renametable() throws Exception {
+ final String table = name.getMethodName() + "1", rename = name.getMethodName() + "2";
+
+ exec("createtable " + table);
+ exec("insert this is a value");
+ exec("renametable " + table + " " + rename);
+ exec("tables", true, rename, true);
+ exec("tables", true, table, false);
+ exec("scan -t " + rename, true, "value", true);
+ exec("deletetable -f " + rename, true);
+ }
+
+ @Test(timeout = 45000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(8, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
+ @Test(timeout = 45000)
+ public void listscans() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+
+ // Should be about a 3 second scan
+ for (int i = 0; i < 6; i++) {
+ exec("insert " + i + " cf cq value", true);
+ }
+ exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
+ Thread thread = new Thread() {
+ public void run() {
+ try {
+ ZooKeeperInstance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers());
+ Connector connector = instance.getConnector("root", new PasswordToken(secret));
+ Scanner s = connector.createScanner(table, Constants.NO_AUTHS);
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> kv : s)
+ ;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
+
+ List<String> scans = new ArrayList<String>();
+ // Try to find the active scan for about 5seconds
+ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
+ String currentScans = exec("listscans", true);
+ String[] lines = currentScans.split("\n");
+ for (int scanOffset = 2; i < lines.length; i++) {
+ String currentScan = lines[scanOffset];
+ if (currentScan.contains(table)) {
+ scans.add(currentScan);
+ }
+ }
+ UtilWaitThread.sleep(100);
+ }
+ thread.join();
+
+ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
+
+ for (String scan : scans) {
+ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
+ String parts[] = scan.split("\\|");
+ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
+ }
+
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void testPertableClasspath() throws Exception {
+ final String table = name.getMethodName();
+
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar");
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar");
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t " + table + " -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline " + table);
+ UtilWaitThread.sleep(500);
+ exec("online " + table);
+
+ exec("table " + table, true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f " + table, true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
+ @Test(timeout = 45000)
+ public void trace() throws Exception {
+ // Make sure to not collide with the "trace" table
+ final String table = name.getMethodName() + "Test";
+
+ exec("trace on", true);
+ exec("createtable " + table, true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f " + table);
+ exec("sleep 1");
+ String trace = exec("trace off");
+ assertTrue(trace.contains("binMutations"));
+ assertTrue(trace.contains("update"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
+
+ private List<String> getFiles(String tableId) throws IOException {
+ output.clear();
+
+ exec("scan -t !METADATA -np -c file -b " + tableId + " -e " + tableId + "~");
+
+ String[] lines = StringUtils.split(output.get(), "\n");
+ output.clear();
+
+ if (0 == lines.length) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
+ }
+
+ private int countFiles(String tableId) throws IOException {
+ return getFiles(tableId).size();
+ }
+
+ private String getTableId(String tableName) throws Exception {
+ ZooKeeperInstance zki = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers());
+ Connector conn = zki.getConnector("root", new PasswordToken(secret));
+
+ for (int i = 0; i < 5; i++) {
+ Map<String,String> nameToId = conn.tableOperations().tableIdMap();
+ if (nameToId.containsKey(tableName)) {
+ return nameToId.get(tableName);
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+
+ fail("Could not find ID for table: " + tableName);
+ // Will never get here
+ return null;
+ }
+
+}