You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2011/11/17 21:21:02 UTC
svn commit: r1203352 - in /incubator/accumulo/trunk: ./ docs/ docs/examples/
docs/src/user_manual/chapters/
src/core/src/main/java/org/apache/accumulo/core/iterators/
src/core/src/main/java/org/apache/accumulo/core/iterators/conf/
src/core/src/main/jav...
Author: kturner
Date: Thu Nov 17 20:21:01 2011
New Revision: 1203352
URL: http://svn.apache.org/viewvc?rev=1203352&view=rev
Log:
ACCUMULO-148 ACCUMULO-154 merge to trunk
Added:
incubator/accumulo/trunk/docs/combiners.html
- copied, changed from r1203347, incubator/accumulo/branches/1.4/docs/combiners.html
incubator/accumulo/trunk/docs/examples/README.combiner
- copied, changed from r1203347, incubator/accumulo/branches/1.4/docs/examples/README.combiner
incubator/accumulo/trunk/src/examples/src/main/java/org/apache/accumulo/examples/combiner/
- copied from r1203347, incubator/accumulo/branches/1.4/src/examples/src/main/java/org/apache/accumulo/examples/combiner/
incubator/accumulo/trunk/src/examples/src/main/java/org/apache/accumulo/examples/combiner/StatsCombiner.java
- copied unchanged from r1203347, incubator/accumulo/branches/1.4/src/examples/src/main/java/org/apache/accumulo/examples/combiner/StatsCombiner.java
Removed:
incubator/accumulo/trunk/docs/aggregation.html
incubator/accumulo/trunk/docs/examples/README.aggregation
incubator/accumulo/trunk/src/examples/src/main/java/org/apache/accumulo/examples/aggregation/
Modified:
incubator/accumulo/trunk/ (props changed)
incubator/accumulo/trunk/docs/config.html
incubator/accumulo/trunk/docs/index.html
incubator/accumulo/trunk/docs/src/user_manual/chapters/analytics.tex
incubator/accumulo/trunk/docs/src/user_manual/chapters/table_configuration.tex
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java
incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java
incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/ServerSideErrorTest.java
Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Nov 17 20:21:01 2011
@@ -1,2 +1,2 @@
/incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383
-/incubator/accumulo/branches/1.4:1201902-1203219
+/incubator/accumulo/branches/1.4:1201902-1203347
Copied: incubator/accumulo/trunk/docs/combiners.html (from r1203347, incubator/accumulo/branches/1.4/docs/combiners.html)
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/docs/combiners.html?p2=incubator/accumulo/trunk/docs/combiners.html&p1=incubator/accumulo/branches/1.4/docs/combiners.html&r1=1203347&r2=1203352&rev=1203352&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/combiners.html (original)
+++ incubator/accumulo/trunk/docs/combiners.html Thu Nov 17 20:21:01 2011
@@ -30,7 +30,7 @@
<p><pre>
Shell - Accumulo Interactive Shell
-- version: 1.4.0-incubating-SNAPSHOT
+- version: 1.5.0-incubating-SNAPSHOT
- instance id: 863fc0d1-3623-4b6c-8c23-7d4fdb1c8a49
-
- type 'help' for a list of available commands
Modified: incubator/accumulo/trunk/docs/config.html
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/docs/config.html?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/docs/config.html (original)
+++ incubator/accumulo/trunk/docs/config.html Thu Nov 17 20:21:01 2011
@@ -1,92 +1,3 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
- <head>
- <title>Configuration Properties</title>
- <link rel='stylesheet' type='text/css' href='documentation.css' media='screen'/>
- </head>
- <body>
- <h1>Configuration Management</h1>
- <p>All accumulo properties have a default value in the source code. Properties can also be set
- in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
- accumulo will choose the property with the highest precedence. This order of precedence is described
- below (from highest to lowest):</p>
- <table>
- <tr><th>Location</th><th>Description</th></tr>
- <tr class='highlight'><td><b>Zookeeper<br/>table properties</td>
- <td>Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml<br/><br/>
- Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
- <pre>config -t TABLE -s PROPERTY=VALUE</pre></td>
- </tr>
- <tr><td><b>Zookeeper<br/>system properties</td>
- <td>System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a 'yes' in the 'Zookeeper Mutable' column in the table below. They are set with the following shell command:
- <pre>config -s PROPERTY=VALUE</pre>
- If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).<br/><br/>
- While most system properties take effect immediately, some require a restart of the process which is indicated in 'Zookeeper Mutable'.</td>
- </tr>
- <tr class='highlight'><td><b>accumulo-site.xml</td>
- <td>Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.<br/><br/>
- Certain properties (indicated by a 'no' in 'Zookeeper Mutable') cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.</td>
- </tr>
- <tr><td><b>Default</td>
- <td>All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.<br/><br/>While the default value is usually optimal, there are cases where a change can increase query and ingest performance.</td>
- </tr>
- </table>
-
- <p>The 'config' command in the shell allows you to view the current system configuration. You can also use the '-t' option to view a table's configuration as below:
-
- <pre>
- $ ./bin/accumulo shell -u root
- Enter current password for 'root'@'ac14': ******
-
- Shell - Accumulo Interactive Shell
- -
- - version: 1.5.0-incubating-SNAPSHOT
- - instance name: ac14
- - instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
- -
- - type 'help' for a list of available commands
- -
- root@ac13> config -t foo
- ---------+---------------------------------------------+------------------------------------------------------
- SCOPE | NAME | VALUE
- ---------+---------------------------------------------+------------------------------------------------------
- default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
- default | table.bloom.enabled ....................... | false
- default | table.bloom.error.rate .................... | 0.5%
- default | table.bloom.hash.type ..................... | murmur
- default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
- default | table.bloom.load.threshold ................ | 1
- default | table.bloom.size .......................... | 1048576
- default | table.cache.block.enable .................. | false
- default | table.cache.index.enable .................. | false
- default | table.compaction.major.everything.at ...... | 19700101000000GMT
- default | table.compaction.major.everything.idle .... | 1h
- default | table.compaction.major.ratio .............. | 1.3
- site | @override .............................. | 1.4
- system | @override .............................. | 1.5
- table | @override .............................. | 1.6
- default | table.compaction.minor.idle ............... | 5m
- default | table.compaction.minor.logs.threshold ..... | 3
- default | table.failures.ignore ..................... | false
- </pre>
-
- <h1>Configuration Properties</h1>
-
<p>Jump to:
<a href='#INSTANCE_PREFIX'>instance.*</a> | <a href='#GENERAL_PREFIX'>general.*</a> | <a href='#MASTER_PREFIX'>master.*</a> | <a href='#TSERV_PREFIX'>tserver.*</a> | <a href='#LOGGER_PREFIX'>logger.*</a> | <a href='#GC_PREFIX'>gc.*</a> | <a href='#MONITOR_PREFIX'>monitor.*</a> | <a href='#TRACE_PREFIX'>trace.*</a> | <a href='#TABLE_PREFIX'>table.*</a> | <a href='#TABLE_CONSTRAINT_PREFIX'>table.constraint.*</a> | <a href='#TABLE_ITERATOR_PREFIX'>table.iterator.*</a> | <a href='#TABLE_LOCALITY_GROUP_PREFIX'>table.group.*</a> </p>
Copied: incubator/accumulo/trunk/docs/examples/README.combiner (from r1203347, incubator/accumulo/branches/1.4/docs/examples/README.combiner)
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/docs/examples/README.combiner?p2=incubator/accumulo/trunk/docs/examples/README.combiner&p1=incubator/accumulo/branches/1.4/docs/examples/README.combiner&r1=1203347&r2=1203352&rev=1203352&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/examples/README.combiner (original)
+++ incubator/accumulo/trunk/docs/examples/README.combiner Thu Nov 17 20:21:01 2011
@@ -25,7 +25,7 @@ tar distribution.
Shell - Accumulo Interactive Shell
-
- - version: 1.4.x-incubating
+ - version: 1.5.x-incubating
- instance name: instance
- instance id: 00000000-0000-0000-0000-000000000000
-
Modified: incubator/accumulo/trunk/docs/index.html
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/docs/index.html?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/docs/index.html (original)
+++ incubator/accumulo/trunk/docs/index.html Thu Nov 17 20:21:01 2011
@@ -25,7 +25,7 @@
<ul>
<li><a href=accumulo_user_manual.pdf>User Manual</a></li>
<li><a href=administration.html>Administration</a></li>
-<li><a href=aggregation.html>Aggregation</a></li>
+<li><a href=combiners.html>Combiners</a></li>
<li><a href=constraints.html>Constraints</a></li>
<li><a href=bulkIngest.html>Bulk Ingest</a></li>
<li><a href=config.html>Configuration</a></li>
Modified: incubator/accumulo/trunk/docs/src/user_manual/chapters/analytics.tex
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/docs/src/user_manual/chapters/analytics.tex?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/docs/src/user_manual/chapters/analytics.tex (original)
+++ incubator/accumulo/trunk/docs/src/user_manual/chapters/analytics.tex Thu Nov 17 20:21:01 2011
@@ -154,33 +154,33 @@ AccumuloOutputFormat.setMaxMutationBuffe
An example of using MapReduce with Accumulo can be found at\\
accumulo/docs/examples/README.mapred
-\section{Aggregating Iterators}
+\section{Combiners}
Many applications can benefit from the ability to aggregate values across common
-keys. This can be done via aggregating iterators and is similar to the Reduce step in
+keys. This can be done via Combiner iterators and is similar to the Reduce step in
MapReduce. This provides the ability to define online, incrementally updated
analytics without the overhead or latency associated with batch-oriented
MapReduce jobs.
All that is needed to aggregate values of a table is to identify the fields over which
values will be grouped, insert mutations with those fields as the key, and configure
-the table with an aggregating iterator that supports the summarization operation
+the table with a combining iterator that supports the summarization operation
desired.
-The only restriction on an aggregating iterator is that the aggregator developer
+The only restriction on an combining iterator is that the combiner developer
should not assume that all values for a given key have been seen, since new
mutations can be inserted at anytime. This precludes using the total number of
values in the aggregation such as when calculating an average, for example.
\subsection{Feature Vectors}
-An interesting use of aggregating iterators within a Accumulo table is to store
+An interesting use of combining iterators within a Accumulo table is to store
feature vectors for use in machine learning algorithms. For example, many
algorithms such as k-means clustering, support vector machines, anomaly detection,
etc. use the concept of a feature vector and the calculation of distance metrics to
learn a particular model. The columns in a Accumulo table can be used to efficiently
store sparse features and their weights to be incrementally updated via the use of an
-aggregating iterator.
+combining iterator.
\section{Statistical Modeling}
Modified: incubator/accumulo/trunk/docs/src/user_manual/chapters/table_configuration.tex
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/docs/src/user_manual/chapters/table_configuration.tex?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/docs/src/user_manual/chapters/table_configuration.tex (original)
+++ incubator/accumulo/trunk/docs/src/user_manual/chapters/table_configuration.tex Thu Nov 17 20:21:01 2011
@@ -293,15 +293,15 @@ table | table.iterator.scan.vers.opt.
\end{verbatim}
\normalsize
-\subsection{Aggregating Iterators}
+\subsection{Combiners}
-Accumulo allows aggregating iterators to be configured on tables and column
-families. When an aggregating iterator is set, the iterator is applied across the values
+Accumulo allows Combiners to be configured on tables and column
+families. When a Combiner is set it is applied across the values
associated with any keys that share rowID, column family, and column qualifier.
This is similar to the reduce step in MapReduce, which applied some function to all
the values associated with a particular key.
-For example, if an aggregating iterator were configured on a table and the following
+For example, if a summing combiner were configured on a table and the following
mutations were inserted:
\small
@@ -320,40 +320,40 @@ rowID1 colfA colqA - 2
\end{verbatim}
\normalsize
-Aggregating iterators can be enabled for a table as follows:
+Combiners can be enabled for a table using the setiter command in the shell. Below is an example.
\small
\begin{verbatim}
-user@myinstance> createtable perDayCounts -a
-day=org.apache.accumulo.core.iterators.aggregation.StringSummation
+root@a14 perDayCounts> setiter -t perDayCounts -p 10 -scan -minc -majc -n daycount
+ -class org.apache.accumulo.core.iterators.user.SummingCombiner
+TypedValueCombiner can interpret Values as a variety of number encodings
+ (VLong, Long, or String) before combining
+----------> set SummingCombiner parameter columns,
+ <col fam>[:<col qual>]{,<col fam>[:<col qual>]} : day
+----------> set SummingCombiner parameter type, <VARNUM|LONG|STRING>: STRING
-user@myinstance perDayCounts> insert row1 day 20080101 1
-user@myinstance perDayCounts> insert row1 day 20080101 1
-user@myinstance perDayCounts> insert row1 day 20080103 1
-user@myinstance perDayCounts> insert row2 day 20080101 1
-user@myinstance perDayCounts> insert row3 day 20080101 1
+root@a14 perDayCounts> insert foo day 20080101 1
+root@a14 perDayCounts> insert foo day 20080101 1
+root@a14 perDayCounts> insert foo day 20080103 1
+root@a14 perDayCounts> insert bar day 20080101 1
+root@a14 perDayCounts> insert bar day 20080101 1
-user@myinstance perDayCounts> scan
-row1 day:20080101 [] 2
-row1 day:20080103 [] 1
-row2 day:20080101 [] 2
+root@a14 perDayCounts> scan
+bar day:20080101 [] 2
+foo day:20080101 [] 2
+foo day:20080103 [] 1
\end{verbatim}
\normalsize
-Accumulo includes the following aggregators:
-\begin{itemize}
-\item{\textbf{LongSummation}: expects values of type long and adds them.}
-\item{\textbf{StringSummation}: expects numbers represented as strings and adds them.}
-\item{\textbf{StringMax}: expects numbers as strings and retains the maximum number inserted.}
-\item{\textbf{StringMin}: expects numbers as strings and retains the minimum number inserted.}
-\end{itemize}
+Accumulo includes some useful Combiners out of the box. To find these look in
+the\\ \textbf{org.apache.accumulo.core.iterators.user} package.
-Additional Aggregators can be added by creating a Java class that implements\\
-\textbf{org.apache.accumulo.core.iterators.aggregation.Aggregator} and adding a jar containing that
-class to Accumulo's lib directory.
+Additional Combiners can be added by creating a Java class that extends\\
+\textbf{org.apache.accumulo.core.iterators.Combiner} and adding a jar containing that
+class to Accumulo's lib/ext directory.
-An example of an aggregator can be found under\\
-accumulo/src/examples/main/java/org/apache/accumulo/examples/aggregation/SortedSetAggregator.java
+An example of a Combiner can be found under\\
+accumulo/src/examples/main/java/org/apache/accumulo/examples/combiner/StatsCombiner.java
\section{Block Cache}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java Thu Nov 17 20:21:01 2011
@@ -17,13 +17,12 @@
package org.apache.accumulo.core.iterators;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.NoSuchElementException;
import org.apache.accumulo.core.client.IteratorSetting;
@@ -33,25 +32,23 @@ import org.apache.accumulo.core.data.Par
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.conf.ColumnSet;
-import org.apache.accumulo.core.iterators.conf.PerColumnIteratorConfig;
+import org.apache.accumulo.core.util.Pair;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
-
-/*
- * A SortedKeyValueIterator that combines the Values for different versions of a Key into a single Value.
- * Combiner will replace one or more versions of a Key and their Values with the most recent Key and a Value which is the result of the reduce method.
+/**
+ * A SortedKeyValueIterator that combines the Values for different versions of a Key into a single Value. Combiner will replace one or more versions of a Key
+ * and their Values with the most recent Key and a Value which is the result of the reduce method.
*
- * Subclasses must implement a reduce method:
- * public Value reduce(Key key, Iterator<Value> iter);
+ * Subclasses must implement a reduce method: public Value reduce(Key key, Iterator<Value> iter);
*
* This reduce method will be passed the most recent Key and an iterator over the Values for all non-deleted versions of that Key.
*/
public abstract class Combiner extends WrappingIterator implements OptionDescriber {
static final Logger log = Logger.getLogger(Combiner.class);
- public static final String COLUMN_PREFIX = "column:";
+ public static final String COLUMNS_OPTION = "columns";
- /*
+ /**
* A Java Iterator that iterates over the Values for a given Key from a source SortedKeyValueIterator.
*/
public static class ValueIterator implements Iterator<Value> {
@@ -59,10 +56,11 @@ public abstract class Combiner extends W
SortedKeyValueIterator<Key,Value> source;
boolean hasNext;
- /*
+ /**
* Constructs an iterator over Values whose Keys are versions of the current topKey of the source SortedKeyValueIterator.
*
- * @param source The SortedKeyValueIterator<Key,Value> from which to read data.
+ * @param source
+ * The SortedKeyValueIterator<Key,Value> from which to read data.
*/
public ValueIterator(SortedKeyValueIterator<Key,Value> source) {
this.source = source;
@@ -74,7 +72,7 @@ public abstract class Combiner extends W
return source.hasTop() && !source.getTopKey().isDeleted() && topKey.equals(source.getTopKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS);
}
- /*
+ /**
* @return <tt>true</tt> if there is another Value
*
* @see java.util.Iterator#hasNext()
@@ -84,7 +82,7 @@ public abstract class Combiner extends W
return hasNext;
}
- /*
+ /**
* @return the next Value
*
* @see java.util.Iterator#next()
@@ -103,7 +101,7 @@ public abstract class Combiner extends W
return topValue;
}
- /*
+ /**
* unsupported
*
* @see java.util.Iterator#remove()
@@ -152,7 +150,7 @@ public abstract class Combiner extends W
private Key workKey = new Key();
- /*
+ /**
* Sets the topKey and topValue based on the top key of the source. If the column of the source top key is in the set of combiners, topKey will be the top key
* of the source and topValue will be the result of the reduce method. Otherwise, topKey and topValue will be unchanged. (They are always set to null before
* this method is called.)
@@ -196,12 +194,14 @@ public abstract class Combiner extends W
}
}
- /*
+ /**
* Reduces a list of Values into a single Value.
*
- * @param key The most recent version of the Key being reduced.
+ * @param key
+ * The most recent version of the Key being reduced.
*
- * @param iter An iterator over the Values for different versions of the key.
+ * @param iter
+ * An iterator over the Values for different versions of the key.
*
* @return The combined Value.
*/
@@ -213,36 +213,87 @@ public abstract class Combiner extends W
public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
super.init(source, options, env);
- List<String> colOpts = new ArrayList<String>();
- for (Entry<String,String> opt : options.entrySet()) {
- if ((opt.getKey().startsWith(COLUMN_PREFIX))) {
- colOpts.add(opt.getKey().substring(COLUMN_PREFIX.length()));
- }
- }
- combiners = new ColumnSet(colOpts);
+ if (!options.containsKey(COLUMNS_OPTION))
+ throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option");
+
+ String encodedColumns = options.get(COLUMNS_OPTION);
+ if (encodedColumns.length() == 0)
+ throw new IllegalArgumentException("The " + COLUMNS_OPTION + " must not be empty");
+
+ combiners = new ColumnSet(Arrays.asList(encodedColumns.split(",")));
}
@Override
public IteratorOptions describeOptions() {
- return new IteratorOptions("comb", "Combiners apply reduce functions to values with identical keys", null, Collections.singletonList("<columnName> null"));
+ return new IteratorOptions(
+ "comb",
+ "Combiners apply reduce functions to values with identical keys",
+ Collections
+ .singletonMap(
+ COLUMNS_OPTION,
+ "<col fam>[:<col qual>]{,<col fam>[:<col qual>]} escape non aplhanum chars using %<hex>."),
+ null);
}
@Override
public boolean validateOptions(Map<String,String> options) {
+ if (!options.containsKey(COLUMNS_OPTION))
+ return false;
+
+ String encodedColumns = options.get(COLUMNS_OPTION);
+ if (encodedColumns.length() == 0)
+ return false;
+
+ for (String columns : encodedColumns.split(",")) {
+ if (!ColumnSet.isValidEncoding(columns))
+ return false;
+ }
+
return true;
}
- /*
- * Adds a column (colf and colq) to an IteratorSetting.
- *
- * @param colf The column family.
- *
- * @param colq The column qualifier (<tt>null</tt> if unspecified).
+ /**
+ * A convenience class for passing column family and column qualifiers
+ */
+ public static class Column extends Pair<Text,Text> {
+
+ public Column(Text columnFamily, Text columnQualifier) {
+ super(columnFamily, columnQualifier);
+ }
+
+ public Column(Text columnFamily) {
+ super(columnFamily, null);
+ }
+
+ public Column(String columnFamily, String columnQualifier) {
+ super(new Text(columnFamily), new Text(columnQualifier));
+ }
+
+ public Column(String columnFamily) {
+ super(new Text(columnFamily), null);
+ }
+
+ }
+
+ /**
+ * A convenience method to set which columns a combiner should be applied to.
*
- * @param is The IteratorSetting to which to add the column parameter.
+ * @param is
+ * iterator settings object to configure
+ * @param columns
+ * a list columns to encode as the value for the combiner column configuration
*/
- public static void addColumn(Text colf, Text colq, IteratorSetting is) {
- String column = PerColumnIteratorConfig.encodeColumns(colf, colq);
- is.addOption(COLUMN_PREFIX + column, "");
+
+ public static void setColumns(IteratorSetting is, List<Column> columns) {
+ String sep = "";
+ StringBuilder sb = new StringBuilder();
+
+ for (Column col : columns) {
+ sb.append(sep);
+ sep = ",";
+ sb.append(ColumnSet.encodeColumns(col.getFirst(), col.getSecond()));
+ }
+
+ is.addOption(COLUMNS_OPTION, sb.toString());
}
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/ColumnSet.java Thu Nov 17 20:21:01 2011
@@ -23,7 +23,9 @@ import java.util.Set;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.iterators.conf.ColumnUtil.ColFamHashKey;
import org.apache.accumulo.core.iterators.conf.ColumnUtil.ColHashKey;
+import org.apache.accumulo.core.util.Pair;
import org.apache.hadoop.io.Text;
+
public class ColumnSet {
private Set<ColFamHashKey> objectsCF;
private Set<ColHashKey> objectsCol;
@@ -40,12 +42,12 @@ public class ColumnSet {
this();
for (String column : objectStrings) {
- PerColumnIteratorConfig pcic = PerColumnIteratorConfig.decodeColumns(column, null);
+ Pair<Text,Text> pcic = ColumnSet.decodeColumns(column);
- if (pcic.getColumnQualifier() == null) {
- add(pcic.getColumnFamily());
+ if (pcic.getSecond() == null) {
+ add(pcic.getFirst());
} else {
- add(pcic.getColumnFamily(), pcic.getColumnQualifier());
+ add(pcic.getFirst(), pcic.getSecond());
}
}
}
@@ -78,4 +80,75 @@ public class ColumnSet {
public boolean isEmpty() {
return objectsCol.size() == 0 && objectsCF.size() == 0;
}
+
+ public static String encodeColumns(Text columnFamily, Text columnQualifier) {
+ StringBuilder sb = new StringBuilder();
+
+ encode(sb, columnFamily);
+ if (columnQualifier != null) {
+ sb.append(':');
+ encode(sb, columnQualifier);
+ }
+
+ return sb.toString();
+ }
+
+ static void encode(StringBuilder sb, Text t) {
+ for (int i = 0; i < t.getLength(); i++) {
+ int b = (0xff & t.getBytes()[i]);
+
+ // very inefficient code
+ if ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' || b == '-') {
+ sb.append((char) b);
+ } else {
+ sb.append('%');
+ sb.append(String.format("%02x", b));
+ }
+ }
+ }
+
+ public static boolean isValidEncoding(String enc) {
+ for (char c : enc.toCharArray()) {
+ boolean validChar = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c == ':' || c == '%';
+ if (!validChar)
+ return false;
+ }
+
+ return true;
+ }
+
+ public static Pair<Text,Text> decodeColumns(String columns) {
+ if (!isValidEncoding(columns))
+ throw new IllegalArgumentException("Invalid encoding " + columns);
+
+ String[] cols = columns.split(":");
+
+ if (cols.length == 1) {
+ return new Pair<Text,Text>(decode(cols[0]), null);
+ } else if (cols.length == 2) {
+ return new Pair<Text,Text>(decode(cols[0]), decode(cols[1]));
+ } else {
+ throw new IllegalArgumentException(columns);
+ }
+ }
+
+ static Text decode(String s) {
+ Text t = new Text();
+
+ byte[] sb = s.getBytes();
+
+ // very inefficient code
+ for (int i = 0; i < sb.length; i++) {
+ if (sb[i] != '%') {
+ t.append(new byte[] {sb[i]}, 0, 1);
+ } else {
+ byte hex[] = new byte[] {sb[++i], sb[++i]};
+ String hs = new String(hex);
+ int b = Integer.parseInt(hs, 16);
+ t.append(new byte[] {(byte) b}, 0, 1);
+ }
+ }
+
+ return t;
+ }
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/conf/PerColumnIteratorConfig.java Thu Nov 17 20:21:01 2011
@@ -58,64 +58,22 @@ public class PerColumnIteratorConfig {
}
private static String encodeColumns(PerColumnIteratorConfig pcic) {
- return encodeColumns(pcic.colf, pcic.colq);
+ return ColumnSet.encodeColumns(pcic.colf, pcic.colq);
}
public static String encodeColumns(Text columnFamily, Text columnQualifier) {
- StringBuilder sb = new StringBuilder();
-
- encode(sb, columnFamily);
- if (columnQualifier != null) {
- sb.append(':');
- encode(sb, columnQualifier);
- }
-
- return sb.toString();
+ return ColumnSet.encodeColumns(columnFamily, columnQualifier);
}
-
- private static void encode(StringBuilder sb, Text t) {
- for (int i = 0; i < t.getLength(); i++) {
- int b = (0xff & t.getBytes()[i]);
-
- // very inefficient code
- if ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' || b == '-') {
- sb.append((char) b);
- } else {
- sb.append('%');
- sb.append(String.format("%02x", b));
- }
- }
- }
-
+
public static PerColumnIteratorConfig decodeColumns(String columns, String className) {
String[] cols = columns.split(":");
if (cols.length == 1) {
- return new PerColumnIteratorConfig(decode(cols[0]), className);
+ return new PerColumnIteratorConfig(ColumnSet.decode(cols[0]), className);
} else if (cols.length == 2) {
- return new PerColumnIteratorConfig(decode(cols[0]), decode(cols[1]), className);
+ return new PerColumnIteratorConfig(ColumnSet.decode(cols[0]), ColumnSet.decode(cols[1]), className);
} else {
throw new IllegalArgumentException(columns);
}
}
-
- private static Text decode(String s) {
- Text t = new Text();
-
- byte[] sb = s.getBytes();
-
- // very inefficient code
- for (int i = 0; i < sb.length; i++) {
- if (sb[i] != '%') {
- t.append(new byte[] {sb[i]}, 0, 1);
- } else {
- byte hex[] = new byte[] {sb[++i], sb[++i]};
- String hs = new String(hex);
- int b = Integer.parseInt(hs, 16);
- t.append(new byte[] {(byte) b}, 0, 1);
- }
- }
-
- return t;
- }
}
Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java (original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java Thu Nov 17 20:21:01 2011
@@ -139,6 +139,10 @@ public class SetIterCommand extends Comm
if (itopts.getName() == null)
throw new IllegalArgumentException(className + " described its default distinguishing name as null");
+ String shortClassName = className;
+ if (className.contains("."))
+ shortClassName = className.substring(className.lastIndexOf('.') + 1);
+
Map<String,String> localOptions = new HashMap<String,String>();
do {
// clean up the overall options that caused things to fail
@@ -152,7 +156,7 @@ public class SetIterCommand extends Comm
String prompt;
if (itopts.getNamedOptions() != null) {
for (Entry<String,String> e : itopts.getNamedOptions().entrySet()) {
- prompt = Shell.repeat("-", 10) + "> set " + className + " parameter " + e.getKey() + ", " + e.getValue() + ": ";
+ prompt = Shell.repeat("-", 10) + "> set " + shortClassName + " parameter " + e.getKey() + ", " + e.getValue() + ": ";
input = reader.readLine(prompt);
if (input == null) {
@@ -171,7 +175,7 @@ public class SetIterCommand extends Comm
reader.printString(Shell.repeat("-", 10) + "> entering options: " + desc + "\n");
input = "start";
while (true) {
- prompt = Shell.repeat("-", 10) + "> set " + className + " option (<name> <value>, hit enter to skip): ";
+ prompt = Shell.repeat("-", 10) + "> set " + shortClassName + " option (<name> <value>, hit enter to skip): ";
input = reader.readLine(prompt);
if (input == null) {
Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java (original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/client/mock/MockConnectorTest.java Thu Nov 17 20:21:01 2011
@@ -84,7 +84,7 @@ public class MockConnectorTest {
c.tableOperations().create(table);
Class<? extends SortedKeyValueIterator<Key,Value>> clazz = SummingCombiner.class;
IteratorSetting is = new IteratorSetting(10, "String Summation", clazz);
- Combiner.addColumn(new Text("day"), null, is);
+ Combiner.setColumns(is, Collections.singletonList(new Combiner.Column("day")));
is.addOption(SummingCombiner.TYPE, SummingCombiner.Type.STRING.name());
c.tableOperations().attachIterator(table, is);
String keys[][] = { {"foo", "day", "20080101"}, {"foo", "day", "20080101"}, {"foo", "day", "20080103"}, {"bar", "day", "20080101"},
Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java (original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/CombinerTest.java Thu Nov 17 20:21:01 2011
@@ -94,6 +94,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.VARNUM.name());
+ opts.put(Combiner.COLUMNS_OPTION, "2");
ai.init(new SortedMapIterator(tm1), opts, null);
ai.seek(new Range(), EMPTY_COL_FAMS, false);
@@ -158,7 +159,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", null);
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.VARNUM.name());
ai.init(new SortedMapIterator(tm1), opts, null);
@@ -225,7 +226,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", "null");
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.LONG.name());
ai.init(new SortedMapIterator(tm1), opts, null);
@@ -295,7 +296,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", null);
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.STRING.name());
ai.init(new SortedMapIterator(tm1), opts, null);
@@ -366,7 +367,7 @@ public class CombinerTest {
Combiner ai = new SummingCombiner();
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", null);
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.STRING.name());
List<SortedKeyValueIterator<Key,Value>> sources = new ArrayList<SortedKeyValueIterator<Key,Value>>(3);
@@ -397,7 +398,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", null);
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.VARNUM.name());
ai.init(new SortedMapIterator(tm1), opts, new DefaultIteratorEnvironment());
@@ -426,7 +427,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", "null");
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.LONG.name());
ai.init(new SortedMapIterator(tm1), opts, new DefaultIteratorEnvironment());
@@ -488,7 +489,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", null);
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, SummingCombiner.Type.VARNUM.name());
ai.init(new SortedMapIterator(tm1), opts, null);
@@ -542,7 +543,7 @@ public class CombinerTest {
Map<String,String> opts = new HashMap<String,String>();
- opts.put(Combiner.COLUMN_PREFIX + "cf001", null);
+ opts.put(Combiner.COLUMNS_OPTION, "cf001");
opts.put(SummingCombiner.TYPE, type);
ai.init(new SortedMapIterator(tm1), opts, null);
Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/ServerSideErrorTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/ServerSideErrorTest.java?rev=1203352&r1=1203351&r2=1203352&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/ServerSideErrorTest.java (original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/ServerSideErrorTest.java Thu Nov 17 20:21:01 2011
@@ -58,7 +58,7 @@ public class ServerSideErrorTest extends
getConnector().tableOperations().create("tt");
IteratorSetting is = new IteratorSetting(5, "Bad Aggregator", BadCombiner.class);
- Combiner.addColumn(new Text("acf"), null, is);
+ Combiner.setColumns(is, Collections.singletonList(new Combiner.Column("acf")));
getConnector().tableOperations().attachIterator("tt", is);
BatchWriter bw = getConnector().createBatchWriter("tt", 1000000, 60000l, 2);