You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bi...@apache.org on 2011/12/12 23:51:16 UTC
svn commit: r1213482 - in /incubator/accumulo/branches/1.4: docs/
docs/src/user_manual/appendices/ docs/src/user_manual/chapters/
src/core/src/main/java/org/apache/accumulo/core/iterators/user/
src/core/src/main/java/org/apache/accumulo/core/util/shell...
Author: billie
Date: Mon Dec 12 22:51:16 2011
New Revision: 1213482
URL: http://svn.apache.org/viewvc?rev=1213482&view=rev
Log:
ACCUMULO-203 tested shell iterator configuration, updated documentation
Added:
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ReqVisFilter.java
- copied, changed from r1212505, incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java
Removed:
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java
Modified:
incubator/accumulo/branches/1.4/docs/config.html
incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/get_shell_commands.sh
incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/shell_commands.tex
incubator/accumulo/branches/1.4/docs/src/user_manual/chapters/table_configuration.tex
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteScanIterCommand.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetScanIterCommand.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java
incubator/accumulo/branches/1.4/test/system/auto/simple/shell.py
Modified: incubator/accumulo/branches/1.4/docs/config.html
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/docs/config.html?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/config.html (original)
+++ incubator/accumulo/branches/1.4/docs/config.html Mon Dec 12 22:51:16 2011
@@ -253,7 +253,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#COUNT'>count</a></b></td>
<td>yes</td>
<td><pre>2</pre></td>
- <td>The miniumum number of threads to use to handle incoming requests.</td>
+ <td>The minimum number of threads to use to handle incoming requests.</td>
</tr>
<tr >
<td>master.tablet.balancer</td>
@@ -466,7 +466,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#COUNT'>count</a></b></td>
<td>yes</td>
<td><pre>16</pre></td>
- <td>The maximum number of concurrent read ahead that will execute. This effectivelylimits the number of long running scans that can run concurrently per tserver.</td>
+ <td>The maximum number of concurrent read ahead that will execute. This effectively limits the number of long running scans that can run concurrently per tserver.</td>
</tr>
<tr >
<td>tserver.scan.files.open.max</td>
@@ -487,7 +487,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#COUNT'>count</a></b></td>
<td>yes</td>
<td><pre>2</pre></td>
- <td>The miniumum number of threads to use to handle incoming requests.</td>
+ <td>The minimum number of threads to use to handle incoming requests.</td>
</tr>
<tr class='highlight'>
<td>tserver.session.idle.max</td>
@@ -532,7 +532,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#PATH'>path</a></b></td>
<td>yes</td>
<td><pre>walogs</pre></td>
- <td>The directory used to store write-ahead logs on the local filesystem</td>
+ <td>The directory used to store write-ahead logs on the local filesystem. It is possible to specify a comma-separated list of directories.</td>
</tr>
<tr >
<td>logger.monitor.fs</td>
@@ -740,7 +740,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#FRACTION'>fraction/percentage</a></b></td>
<td>yes</td>
<td><pre>3</pre></td>
- <td>minimum ratio of total input size to maximum input file size for running a major compaction. When adjusting this property you may want to also adjust table.file.max. Want to avoid the situation where only merging minor compactions occurr.</td>
+ <td>minimum ratio of total input size to maximum input file size for running a major compaction. When adjusting this property you may want to also adjust table.file.max. Want to avoid the situation where only merging minor compactions occur.</td>
</tr>
<tr >
<td>table.compaction.minor.idle</td>
@@ -782,7 +782,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#MEMORY'>memory</a></b></td>
<td>yes</td>
<td><pre>128K</pre></td>
- <td>Determines how large index blocks can be in files that support multilevel indexesThe maximum value for this is 2147483647</td>
+ <td>Determines how large index blocks can be in files that support multilevel indexes. The maximum value for this is 2147483647</td>
</tr>
<tr >
<td>table.file.compress.type</td>
@@ -838,7 +838,7 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td><b><a href='#MEMORY'>memory</a></b></td>
<td>yes</td>
<td><pre>1G</pre></td>
- <td>When combined size of mapfiles exceeds this amount a tablet is split.</td>
+ <td>When combined size of files exceeds this amount a tablet is split.</td>
</tr>
<tr >
<td>table.walog.enabled</td>
Modified: incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/get_shell_commands.sh
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/get_shell_commands.sh?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/get_shell_commands.sh (original)
+++ incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/get_shell_commands.sh Mon Dec 12 22:51:16 2011
@@ -16,6 +16,22 @@
# limitations under the License.
+echo "% Licensed to the Apache Software Foundation (ASF) under one or more
+% contributor license agreements. See the NOTICE file distributed with
+% this work for additional information regarding copyright ownership.
+% The ASF licenses this file to You under the Apache License, Version 2.0
+% (the \"License\"); you may not use this file except in compliance with
+% the License. You may obtain a copy of the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an \"AS IS\" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+"
+
echo "\\chapter{Shell Commands}"
echo
echo "\\begin{alltt}"
Modified: incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/shell_commands.tex
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/shell_commands.tex?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/shell_commands.tex (original)
+++ incubator/accumulo/branches/1.4/docs/src/user_manual/appendices/shell_commands.tex Mon Dec 12 22:51:16 2011
@@ -1,4 +1,3 @@
-
% Licensed to the Apache Software Foundation (ASF) under one or more
% contributor license agreements. See the NOTICE file distributed with
% this work for additional information regarding copyright ownership.
@@ -248,19 +247,23 @@
usage: egrep <regex>{ <regex>} [-?] [-b <start-row>] [-c
<{<columnfamily>[:<columnqualifier>]}>] [-e <end-row>] [-f <int>]
- [-np] [-s <comma-separated-authorizations>] [-st] [-t <arg>]
+ [-np] [-nt <arg>] [-s <comma-separated-authorizations>] [-st] [-t
+ <table>]
description: searches each row, column family, column qualifier and value, in
- parallel, on the server side (using a java regular expression)
+ parallel, on the server side (using a java Matcher, so put .*
+ before and after your term if you're not matching the whole
+ element)
-?,--help display this help
-b,--begin-row <start-row> begin row (inclusive)
-c,--columns <{<columnfamily>[:<columnqualifier>]}> comma-separated columns
-e,--end-row <end-row> end row (inclusive)
-f,--show few <int> Only shows certain amount of characters
-np,--no-pagination disables pagination of output
+ -nt,--num-threads <arg> num threads
-s,--scan-authorizations <comma-separated-authorizations> scan authorizations
(all user auths are used if this argument is not specified)
-st,--show-timestamps enables displaying timestamps
- -t,--num-threads <arg> num threads
+ -t,--tableName <table> table to grep through
\textbf{execfile}
@@ -334,7 +337,8 @@
usage: grep <term>{ <term>} [-?] [-b <start-row>] [-c
<{<columnfamily>[:<columnqualifier>]}>] [-e <end-row>] [-f <int>]
- [-np] [-s <comma-separated-authorizations>] [-st] [-t <arg>]
+ [-np] [-nt <arg>] [-s <comma-separated-authorizations>] [-st] [-t
+ <table>]
description: searches each row, column family, column qualifier and value in a
table for a substring (not a regular expression), in parallel, on
the server side
@@ -344,10 +348,11 @@
-e,--end-row <end-row> end row (inclusive)
-f,--show few <int> Only shows certain amount of characters
-np,--no-pagination disables pagination of output
+ -nt,--num-threads <arg> num threads
-s,--scan-authorizations <comma-separated-authorizations> scan authorizations
(all user auths are used if this argument is not specified)
-st,--show-timestamps enables displaying timestamps
- -t,--num-threads <arg> num threads
+ -t,--tableName <table> table to grep through
\textbf{help}
@@ -390,7 +395,7 @@
usage: listscans [-?] [-np] [-ts <tablet server>]
description: list what scans are currently running in accumulo. See the
- org.apache.accumulo.core.client.admin.ActiveScan javadoc for more information
+ accumulo.core.client.admin.ActiveScan javadoc for more information
about columns.
-?,--help display this help
-np,--no-pagination disables pagination of output
@@ -540,8 +545,8 @@
\textbf{setiter}
- usage: setiter [-?] -ageoff | -agg | -class <name> | -nolabel | -regex | -vers
- [-majc] [-minc] [-n <itername>] -p <pri> [-scan] [-t <table>]
+ usage: setiter [-?] -ageoff | -agg | -class <name> | -regex | -reqvis | -vers
+ [-majc] [-minc] [-n <itername>] -p <pri> [-scan] [-t <table>]
description: sets a table-specific iterator
-?,--help display this help
-ageoff,--ageoff an aging off type
@@ -550,26 +555,26 @@
-majc,--major-compaction applied at major compaction
-minc,--minor-compaction applied at minor compaction
-n,--name <itername> iterator to set
- -nolabel,--no-label a no-labeling type
-p,--priority <pri> the order in which the iterator is applied
-regex,--regular-expression a regex matching type
+ -reqvis,--require-visibility a type that omits entries with empty visibilities
-scan,--scan-time applied at scan time
-t,--table <table> tableName
-vers,--version a versioning type
\textbf{setscaniter}
- usage: setscaniter [-?] -ageoff | -agg | -class <name> | -nolabel | -regex |
- -vers [-n <itername>] -p <pri> [-t <table>]
+ usage: setscaniter [-?] -ageoff | -agg | -class <name> | -regex | -reqvis | -vers
+ [-n <itername>] -p <pri> [-t <table>]
description: sets a table-specific scan iterator for this shell session
-?,--help display this help
-ageoff,--ageoff an aging off type
-agg,--aggregator an aggregating type
-class,--class-name <name> a java class type
-n,--name <itername> iterator to set
- -nolabel,--no-label a no-labeling type
-p,--priority <pri> the order in which the iterator is applied
-regex,--regular-expression a regex matching type
+ -reqvis,--require-visibility a type that omits entries with empty visibilities
-t,--table <table> tableName
-vers,--version a versioning type
Modified: incubator/accumulo/branches/1.4/docs/src/user_manual/chapters/table_configuration.tex
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/docs/src/user_manual/chapters/table_configuration.tex?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/docs/src/user_manual/chapters/table_configuration.tex (original)
+++ incubator/accumulo/branches/1.4/docs/src/user_manual/chapters/table_configuration.tex Mon Dec 12 22:51:16 2011
@@ -149,9 +149,9 @@ Some useful Iterators are provided with
\small
\begin{verbatim}
-usage: setiter [-?] -agg | -class <name> | -filter | -nolabel |
--regex | -vers [-majc] [-minc] [-n <itername>] -p <pri> [-scan]
-[-t <table>]
+usage: setiter [-?] -ageoff | -agg | -class <name> | -regex |
+-reqvis | -vers [-majc] [-minc] [-n <itername>] -p <pri>
+[-scan] [-t <table>]
user@myinstance mytable> setiter -t mytable -scan -p 10 -n myiter
\end{verbatim}
Copied: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ReqVisFilter.java (from r1212505, incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java)
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ReqVisFilter.java?p2=incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ReqVisFilter.java&p1=incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java&r1=1212505&r2=1213482&rev=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/NoVisFilter.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/ReqVisFilter.java Mon Dec 12 22:51:16 2011
@@ -24,7 +24,7 @@ import org.apache.accumulo.core.security
/**
* A Filter that matches entries with a non-empty ColumnVisibility.
*/
-public class NoVisFilter extends Filter {
+public class ReqVisFilter extends Filter {
@Override
public boolean accept(Key k, Value v) {
@@ -35,8 +35,8 @@ public class NoVisFilter extends Filter
@Override
public IteratorOptions describeOptions() {
IteratorOptions io = super.describeOptions();
- io.setName("novis");
- io.setDescription("NoVisFilter hides entries without a visibility label");
+ io.setName("reqvis");
+ io.setDescription("ReqVisFilter hides entries without a visibility label");
return io;
}
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java Mon Dec 12 22:51:16 2011
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -44,6 +45,7 @@ import org.apache.accumulo.core.client.A
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.mock.MockInstance;
@@ -163,7 +165,7 @@ public class Shell {
private AuthInfo credentials;
private Class<? extends Formatter> formatterClass = DefaultFormatter.class;
private Class<? extends Formatter> binaryFormatterClass = BinaryFormatter.class;
- public Map<String,Map<String,Map<String,String>>> scanIteratorOptions = new HashMap<String,Map<String,Map<String,String>>>();
+ public Map<String,List<IteratorSetting>> scanIteratorOptions = new HashMap<String,List<IteratorSetting>>();
private Token rootToken;
public final Map<String,Command> commandFactory = new TreeMap<String,Command>();
@@ -458,11 +460,13 @@ public class Shell {
sb.append("- Active formatter class: ").append(formatterClass.getSimpleName()).append("\n");
}
if (!scanIteratorOptions.isEmpty()) {
- for (Entry<String,Map<String,Map<String,String>>> entry : scanIteratorOptions.entrySet()) {
+ for (Entry<String,List<IteratorSetting>> entry : scanIteratorOptions.entrySet()) {
sb.append("- Session scan iterators for table ").append(entry.getKey()).append(":\n");
- for (Entry<String,Map<String,String>> namedEntry : entry.getValue().entrySet()) {
- sb.append("- Iterator ").append(namedEntry.getKey()).append(" options:\n");
- for (Entry<String,String> optEntry : namedEntry.getValue().entrySet()) {
+ for (IteratorSetting setting : entry.getValue()) {
+ sb.append("- Iterator ").append(setting.getName()).append(" options:\n");
+ sb.append("- ").append("iteratorPriority").append(" = ").append(setting.getPriority()).append("\n");
+ sb.append("- ").append("iteratorClassName").append(" = ").append(setting.getIteratorClass()).append("\n");
+ for (Entry<String,String> optEntry : setting.getProperties().entrySet()) {
sb.append("- ").append(optEntry.getKey()).append(" = ").append(optEntry.getValue()).append("\n");
}
}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteScanIterCommand.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteScanIterCommand.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteScanIterCommand.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteScanIterCommand.java Mon Dec 12 22:51:16 2011
@@ -16,8 +16,10 @@
*/
package org.apache.accumulo.core.util.shell.commands;
-import java.util.Map;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.Shell.Command;
@@ -44,20 +46,30 @@ public class DeleteScanIterCommand exten
}
if (cl.hasOption(allOpt.getOpt())) {
- Map<String,Map<String,String>> tableIterators = shellState.scanIteratorOptions.remove(tableName);
- if (tableIterators == null)
+ List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.remove(tableName);
+ if (tableScanIterators == null)
Shell.log.info("No scan iterators set on table " + tableName);
else
- Shell.log.info("Removed the following scan iterators from table " + tableName + ":" + tableIterators.keySet());
+ Shell.log.info("Removed the following scan iterators from table " + tableName + ":" + tableScanIterators);
} else if (cl.hasOption(nameOpt.getOpt())) {
String name = cl.getOptionValue(nameOpt.getOpt());
- Map<String,Map<String,String>> tableIterators = shellState.scanIteratorOptions.get(tableName);
- if (tableIterators != null) {
- Map<String,String> options = tableIterators.remove(name);
- if (options == null)
+ List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(tableName);
+ if (tableScanIterators != null) {
+ boolean found = false;
+ for (Iterator<IteratorSetting> iter = tableScanIterators.iterator(); iter.hasNext();) {
+ if (iter.next().getName().equals(name)) {
+ iter.remove();
+ found = true;
+ break;
+ }
+ }
+ if (!found)
Shell.log.info("No iterator named " + name + " found for table " + tableName);
- else
- Shell.log.info("Removed scan iterator " + name + " from table " + tableName);
+ else {
+ Shell.log.info("Removed scan iterator " + name + " from table " + tableName + " (" + shellState.scanIteratorOptions.get(tableName).size() + " left)");
+ if (shellState.scanIteratorOptions.get(tableName).size() == 0)
+ shellState.scanIteratorOptions.remove(tableName);
+ }
} else {
Shell.log.info("No iterator named " + name + " found for table " + tableName);
}
@@ -97,4 +109,4 @@ public class DeleteScanIterCommand exten
public int numArgs() {
return 0;
}
-}
\ No newline at end of file
+}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/ScanCommand.java Mon Dec 12 22:51:16 2011
@@ -17,8 +17,7 @@
package org.apache.accumulo.core.util.shell.commands;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.List;
import java.util.Map.Entry;
import org.apache.accumulo.core.client.AccumuloException;
@@ -96,23 +95,20 @@ public class ScanCommand extends Command
}
protected void addScanIterators(Shell shellState, Scanner scanner, String tableName) {
- Map<String,Map<String,String>> tableIterators = shellState.scanIteratorOptions.get(shellState.getTableName());
- if (tableIterators == null)
+ List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(shellState.getTableName());
+ if (tableScanIterators == null) {
+ Shell.log.debug("Found no scan iterators to set");
return;
+ }
+ Shell.log.debug("Found " + tableScanIterators.size() + " scan iterators to set");
- for (Entry<String,Map<String,String>> entry : tableIterators.entrySet()) {
- Map<String,String> options = new HashMap<String,String>(entry.getValue());
- String className = options.remove("iteratorClassName");
- int priority = Integer.parseInt(options.remove("iteratorPriority"));
- String name = entry.getKey();
-
- Shell.log.debug("Setting scan iterator " + name + " at priority " + priority + " using class name " + className);
- IteratorSetting si = new IteratorSetting(priority, name, className);
- for (Entry<String,String> option : options.entrySet()) {
- Shell.log.debug("Setting option for " + name + ": " + option.getKey() + "=" + option.getValue());
+ for (IteratorSetting setting : tableScanIterators) {
+ Shell.log.debug("Setting scan iterator " + setting.getName() + " at priority " + setting.getPriority() + " using class name "
+ + setting.getIteratorClass());
+ for (Entry<String,String> option : setting.getProperties().entrySet()) {
+ Shell.log.debug("Setting option for " + setting.getName() + ": " + option.getKey() + "=" + option.getValue());
}
- si.addOptions(options.entrySet());
- scanner.addScanIterator(si);
+ scanner.addScanIterator(setting);
}
}
@@ -195,4 +191,4 @@ public class ScanCommand extends Command
public int numArgs() {
return 0;
}
-}
\ No newline at end of file
+}
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java Mon Dec 12 22:51:16 2011
@@ -36,7 +36,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.aggregation.Aggregator;
import org.apache.accumulo.core.iterators.user.AgeOffFilter;
-import org.apache.accumulo.core.iterators.user.NoVisFilter;
+import org.apache.accumulo.core.iterators.user.ReqVisFilter;
import org.apache.accumulo.core.iterators.user.RegExFilter;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.util.shell.Shell;
@@ -53,7 +53,7 @@ import org.apache.commons.cli.Options;
public class SetIterCommand extends Command {
private Option tableOpt, mincScopeOpt, majcScopeOpt, scanScopeOpt, nameOpt, priorityOpt;
- private Option aggTypeOpt, ageoffTypeOpt, regexTypeOpt, versionTypeOpt, nolabelTypeOpt, classnameTypeOpt;
+ private Option aggTypeOpt, ageoffTypeOpt, regexTypeOpt, versionTypeOpt, reqvisTypeOpt, classnameTypeOpt;
public int execute(String fullCommand, CommandLine cl, Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
IOException, ShellCommandException {
@@ -84,8 +84,8 @@ public class SetIterCommand extends Comm
classname = AgeOffFilter.class.getName();
else if (cl.hasOption(versionTypeOpt.getOpt()))
classname = VersioningIterator.class.getName();
- else if (cl.hasOption(nolabelTypeOpt.getOpt()))
- classname = NoVisFilter.class.getName();
+ else if (cl.hasOption(reqvisTypeOpt.getOpt()))
+ classname = ReqVisFilter.class.getName();
if (!shellState.getConnector().instanceOperations().testClassLoad(classname, SortedKeyValueIterator.class.getName()))
throw new ShellCommandException(ErrorCode.INITIALIZATION_FAILURE, "Servers are unable to load " + classname + " as type "
@@ -236,14 +236,14 @@ public class SetIterCommand extends Comm
aggTypeOpt = new Option("agg", "aggregator", false, "an aggregating type");
regexTypeOpt = new Option("regex", "regular-expression", false, "a regex matching type");
versionTypeOpt = new Option("vers", "version", false, "a versioning type");
- nolabelTypeOpt = new Option("nolabel", "no-label", false, "a blank visibility omitting type");
+ reqvisTypeOpt = new Option("reqvis", "require-visibility", false, "a type that omits entries with empty visibilities");
ageoffTypeOpt = new Option("ageoff", "ageoff", false, "an aging off type");
typeGroup.addOption(classnameTypeOpt);
typeGroup.addOption(aggTypeOpt);
typeGroup.addOption(regexTypeOpt);
typeGroup.addOption(versionTypeOpt);
- typeGroup.addOption(nolabelTypeOpt);
+ typeGroup.addOption(reqvisTypeOpt);
typeGroup.addOption(ageoffTypeOpt);
typeGroup.setRequired(true);
Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetScanIterCommand.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetScanIterCommand.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetScanIterCommand.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetScanIterCommand.java Mon Dec 12 22:51:16 2011
@@ -17,16 +17,20 @@
package org.apache.accumulo.core.util.shell.commands;
import java.io.IOException;
-import java.util.HashMap;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.shell.Shell;
import org.apache.accumulo.core.util.shell.ShellCommandException;
import org.apache.accumulo.core.util.shell.ShellCommandException.ErrorCode;
@@ -44,20 +48,31 @@ public class SetScanIterCommand extends
@Override
protected void setTableProperties(CommandLine cl, Shell shellState, String tableName, int priority, Map<String,String> options, String classname, String name)
- throws AccumuloException, AccumuloSecurityException, ShellCommandException {
- // instead of setting table properties, just put the options
- // in a map to use at scan time
+ throws AccumuloException, AccumuloSecurityException, ShellCommandException, TableNotFoundException {
+ // instead of setting table properties, just put the options in a list to use at scan time
if (!shellState.getConnector().instanceOperations().testClassLoad(classname, SortedKeyValueIterator.class.getName()))
throw new ShellCommandException(ErrorCode.INITIALIZATION_FAILURE, "Servers are unable to load " + classname + " as type "
+ SortedKeyValueIterator.class.getName());
- options.put("iteratorClassName", classname);
- options.put("iteratorPriority", Integer.toString(priority));
- Map<String,Map<String,String>> tableIterators = shellState.scanIteratorOptions.get(tableName);
- if (tableIterators == null) {
- tableIterators = new HashMap<String,Map<String,String>>();
- shellState.scanIteratorOptions.put(tableName, tableIterators);
+ List<IteratorSetting> tableScanIterators = shellState.scanIteratorOptions.get(tableName);
+ if (tableScanIterators == null) {
+ tableScanIterators = new ArrayList<IteratorSetting>();
+ shellState.scanIteratorOptions.put(tableName, tableScanIterators);
+ }
+ IteratorSetting setting = new IteratorSetting(priority, name, classname);
+ setting.addOptions(options);
+
+ // initialize a scanner to ensure the new setting does not conflict with existing settings
+ String user = shellState.getConnector().whoami();
+ Authorizations auths = shellState.getConnector().securityOperations().getUserAuthorizations(user);
+ Scanner scanner = shellState.getConnector().createScanner(tableName, auths);
+ for (IteratorSetting s : tableScanIterators) {
+ scanner.addScanIterator(s);
}
- tableIterators.put(name, options);
+ scanner.addScanIterator(setting);
+
+ // if no exception has been thrown, it's safe to add it to the list
+ tableScanIterators.add(setting);
+ Shell.log.debug("Scan iterators :" + shellState.scanIteratorOptions.get(tableName));
}
@Override
@@ -87,4 +102,4 @@ public class SetScanIterCommand extends
return modifiedOptions;
}
-}
\ No newline at end of file
+}
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/client/impl/ScannerOptionsTest.java Mon Dec 12 22:51:16 2011
@@ -17,8 +17,10 @@
package org.apache.accumulo.core.client.impl;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.iterators.DebugIterator;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.junit.Test;
@@ -40,4 +42,18 @@ public class ScannerOptionsTest {
options.removeScanIterator("NAME");
assertEquals(0, options.serverSideIteratorList.size());
}
+
+ @Test
+ public void testIteratorConflict() {
+ ScannerOptions options = new ScannerOptions();
+ options.addScanIterator(new IteratorSetting(1, "NAME", DebugIterator.class));
+ try {
+ options.addScanIterator(new IteratorSetting(2, "NAME", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ try {
+ options.addScanIterator(new IteratorSetting(1, "NAME2", DebugIterator.class));
+ fail();
+ } catch (IllegalArgumentException e) {}
+ }
}
Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java (original)
+++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java Mon Dec 12 22:51:16 2011
@@ -323,7 +323,7 @@ public class FilterTest extends TestCase
}
assertTrue(tm.size() == 1000);
- Filter filter = new NoVisFilter();
+ Filter filter = new ReqVisFilter();
filter.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
filter.seek(new Range(), EMPTY_COL_FAMS, false);
int size = size(filter);
Modified: incubator/accumulo/branches/1.4/test/system/auto/simple/shell.py
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/test/system/auto/simple/shell.py?rev=1213482&r1=1213481&r2=1213482&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/test/system/auto/simple/shell.py (original)
+++ incubator/accumulo/branches/1.4/test/system/auto/simple/shell.py Mon Dec 12 22:51:16 2011
@@ -37,6 +37,7 @@ class ShellTest(TestUtilsMixin,unittest.
def runTest(self):
self.setIterTest()
+ self.setScanIterTest()
self.aggTest()
self.iteratorsTest()
self.createtableTestSplits()
@@ -75,7 +76,7 @@ class ShellTest(TestUtilsMixin,unittest.
input = 'setiter -t setitertest -n mymin -scan -p 10 -class org.apache.accumulo.core.iterators.user.MinCombiner\ncf2\nSTRING\n'
out,err,code = self.rootShell(self.masterHost(),input)
self.failUnless(out.find("IllegalArgumentException") >= 0,
- "Was able to configure same prioritytwice")
+ "Was able to configure same priority twice")
input = 'setiter -t setitertest -n mymin -scan -p 11 -class org.apache.accumulo.core.iterators.user.MinCombiner\ncf2\nSTRING\n'
out,err,code = self.rootShell(self.masterHost(),input)
self.processResult(out, err, code)
@@ -83,8 +84,42 @@ class ShellTest(TestUtilsMixin,unittest.
out,err,code = self.rootShell(self.masterHost(),input)
self.processResult(out, err, code)
self.failIf(out.find("row1 cf1:cq [] 30") == -1 or out.find("row1 cf2:cq [] 10") == -1,
- "Config Failed: combining failed")
-
+ "SetIter Failed: combining failed")
+
+ def setScanIterTest(self):
+ input = 'createtable setscanitertest\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ input = 'table setscanitertest\ninsert row cf cq val1\ninsert row cf cq val2\nscan -np\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ self.failIf(out.find("row cf:cq [] val1") == 1 or out.find("row cf:cq [] val2") == -1,
+ "SetScanIter Failed: default versioning failed")
+ input = 'setscaniter -t setscanitertest -n vers -p 20 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\ntable setscanitertest\nscan -np\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ self.failIf(out.find("row cf:cq [] val1") == -1 or out.find("row cf:cq [] val2") == -1,
+ "SetScanIter Failed: versioning override failed")
+ input = 'setscaniter -t setscanitertest -n vers -p 20 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\ndeletescaniter -t setscanitertest -n vers\ntable setscanitertest\nscan -np\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ self.failIf(out.find("row cf:cq [] val1") == 1 or out.find("row cf:cq [] val2") == -1,
+ "SetScanIter Failed: deletescaniter (single) failed")
+ input = 'setscaniter -t setscanitertest -n vers -p 20 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\ndeletescaniter -t setscanitertest -a\ntable setscanitertest\nscan -np\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ self.failIf(out.find("row cf:cq [] val1") == 1 or out.find("row cf:cq [] val2") == -1,
+ "SetScanIter Failed: deletescaniter (all) failed")
+ input = 'setscaniter -t setscanitertest -n vers -p 20 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\nsetscaniter -t setscanitertest -n vers -p 10 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ self.failUnless(out.find("IllegalArgumentException") >= 0,
+ "Was able to configure same iter name twice")
+ input = 'setscaniter -t setscanitertest -n vers -p 20 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\nsetscaniter -t setscanitertest -n vers2 -p 20 -class org.apache.accumulo.core.iterators.user.VersioningIterator\n2\n'
+ out,err,code = self.rootShell(self.masterHost(),input)
+ self.processResult(out, err, code)
+ self.failUnless(out.find("IllegalArgumentException") >= 0,
+ "Was able to configure same priority twice")
def aggTest(self):
input = 'createtable aggtest\nsetiter -t aggtest -n myagg -scan -p 10 -class org.apache.accumulo.core.iterators.user.SummingCombiner\ns\nSTRING\n\nquit\n'