You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/10 23:15:58 UTC
[01/16] git commit: ACCUMULO-1936 Make testInstanceOperations a more
reliable test.
Repository: accumulo
Updated Branches:
refs/heads/1.5.2-SNAPSHOT bc9cee0c0 -> 4cc25caa0
refs/heads/1.6.0-SNAPSHOT 4223d6681 -> da7f937b1
refs/heads/master 977d41d7b -> 389003fde
ACCUMULO-1936 Make testInstanceOperations a more reliable test.
Avoid pulling scans for users other than root and compactions for
compactions over tables other than the one we're looking at. Slow down
the slow iterator even more.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/72395e01
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/72395e01
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/72395e01
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: 72395e010d3f495507d03664057a63c3d690a206
Parents: e13c273
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:26:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/proxy/SimpleTest.java | 110 +++++++++++++------
1 file changed, 75 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/72395e01/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 5573d20..77e7b9e 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServer;
@@ -102,7 +103,8 @@ import org.junit.rules.TemporaryFolder;
* Call every method on the proxy and try to verify that it works.
*/
public class SimpleTest {
-
+ private static final Logger log = Logger.getLogger(SimpleTest.class);
+
public static TemporaryFolder folder = new TemporaryFolder();
public static final String TABLE_TEST = "test";
@@ -704,7 +706,7 @@ public class SimpleTest {
assertEquals(0, entries.results.size());
}
- @Test(timeout = 10000)
+ @Test(timeout = 60000)
public void testInstanceOperations() throws Exception {
int tservers = 0;
for (String tserver : client.getTabletServers(creds)) {
@@ -745,12 +747,14 @@ public class SimpleTest {
// create a table that's very slow, so we can look for scans/compactions
client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ // Should take 5 seconds to read every record
+ for (int i = 0; i < 20; i++) {
+ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
+ }
+
+ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
// scan
Thread t = new Thread() {
@@ -768,28 +772,45 @@ public class SimpleTest {
}
};
t.start();
+
// look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
+ List<ActiveScan> scans = new ArrayList<ActiveScan>();
+ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
+ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
+ for (ActiveScan scan : scansForServer) {
+ if ("root".equals(scan.getUser())) {
+ scans.add(scan);
+ }
+ }
+
if (!scans.isEmpty())
- break loop;
+ break;
UtilWaitThread.sleep(10);
}
}
t.join();
- assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
+ assertFalse(scans.isEmpty());
+ boolean found = false;
+ Map<String,String> map = null;
+ for (int i = 0; i < scans.size() && !found; i++) {
+ ActiveScan scan = scans.get(i);
+ if ("root".equals(scan.getUser())) {
+ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
+ assertEquals(ScanType.SINGLE, scan.getType());
+ assertEquals("slow", scan.getTable());
+
+ map = client.tableIdMap(creds);
+ assertEquals(map.get("slow"), scan.getExtent().tableId);
+ assertTrue(scan.getExtent().endRow == null);
+ assertTrue(scan.getExtent().prevEndRow == null);
+ found = true;
+ }
+ }
+
+ assertTrue("Could not find a scan against the 'slow' table", found);
+
// start a compaction
t = new Thread() {
@Override
@@ -803,27 +824,46 @@ public class SimpleTest {
}
};
t.start();
-
+
+ final String desiredTableId = map.get("slow");
+
// try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
+ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
+ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
+ // Iterate over the tservers
for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
+ // And get the compactions on each
+ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
+ for (ActiveCompaction compact : compactionsOnServer) {
+ // There might be other compactions occurring (e.g. on METADATA) in which
+ // case we want to prune out those that aren't for our slow table
+ if (desiredTableId.equals(compact.getExtent().tableId)) {
+ compactions.add(compact);
+ }
+ }
+
+ // If we found a compaction for the table we wanted, so we can stop looking
if (!compactions.isEmpty())
- break loop2;
+ break;
}
UtilWaitThread.sleep(10);
}
t.join();
+
// verify the compaction information
assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
+ for (ActiveCompaction c : compactions) {
+ if (desiredTableId.equals(c.getExtent().tableId)) {
+ assertTrue(c.inputFiles.isEmpty());
+ assertEquals(CompactionType.MINOR, c.getType());
+ assertEquals(CompactionReason.USER, c.getReason());
+ assertEquals("", c.localityGroup);
+ assertTrue(c.outputFile.contains("default_tablet"));
+
+ return;
+ }
+ }
+ fail("Expection to find running compaction for table 'slow' but did not find one");
}
@Test
@@ -1178,5 +1218,5 @@ public class SimpleTest {
accumulo.stop();
folder.delete();
}
-
-}
+
+}
\ No newline at end of file
[07/16] git commit: ACCUMULO-2433 Add a way to still use the exec
methods but provide a custom error message on failure.
Posted by el...@apache.org.
ACCUMULO-2433 Add a way to still use the exec methods but provide a custom error message on failure.
Fix some tests that I've seen fail to provide better messages on failure to help future debugging
because I can't get them reproduced locally in any reasonable timeframe.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4cc25caa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4cc25caa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4cc25caa
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 4cc25caa07ea5ca49e4d81f9e5a22908e45bba59
Parents: 72395e0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:59:09 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 17:13:22 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/test/ShellServerTest.java | 128 ++++++++++++++-----
1 file changed, 99 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4cc25caa/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index e2ebbcf..d66f509 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -31,6 +31,10 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -53,6 +57,7 @@ import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.server.trace.TraceServer;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -85,7 +90,19 @@ public class ShellServerTest {
}
}
+ private static abstract class ErrorMessageCallback {
+ public abstract String getErrorMessage();
+ }
+
+ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
+ private static final String empty = "";
+ public String getErrorMessage() {
+ return empty;
+ }
+ }
+
private static final Logger log = Logger.getLogger(ShellServerTest.class);
+ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
@@ -104,36 +121,51 @@ public class ShellServerTest {
}
String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit("", true);
+ assertGoodExit("", true, callback);
else
- assertBadExit("", true);
+ assertBadExit("", true, callback);
return result;
}
String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
}
String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
+ assertGoodExit(expectString, stringPresent, callback);
else
- assertBadExit(expectString, stringPresent);
+ assertBadExit(expectString, stringPresent, callback);
return result;
}
- void assertGoodExit(String s, boolean stringPresent) {
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
- assertEquals(0, shell.getExitCode());
+ if (0 != shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(0, shell.getExitCode());
+ }
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
- void assertBadExit(String s, boolean stringPresent) {
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
if (s.length() > 0)
@@ -409,7 +441,16 @@ public class ShellServerTest {
final String table = name.getMethodName();
// addauths
exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
+ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
+ public String getErrorMessage() {
+ try {
+ Connector c = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getConnector("root", new PasswordToken(secret));
+ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+ });
exec("addauths -s foo,bar", true);
exec("getauths", true, "foo", true);
exec("getauths", true, "bar", true);
@@ -486,10 +527,10 @@ public class ShellServerTest {
exec("flush -w");
exec("insert n 1 2 3");
exec("flush -w");
- oldCount = countFiles(tableId);
+ List<String> oldFiles = getFiles(tableId);
// at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
+ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
// compact some data:
exec("compact -b g -e z -w");
@@ -564,7 +605,8 @@ public class ShellServerTest {
exec("addsplits row5 row7");
make10();
exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
+ List<String> files = getFiles(tableId);
+ assertEquals("Found the following files: " + files, 3, files.size());
exec("deleterows -t " + table + " -b row5 -e row7", true);
assertEquals(2, countFiles(tableId));
exec("deletetable -f " + table);
@@ -767,12 +809,13 @@ public class ShellServerTest {
final String table = name.getMethodName();
exec("createtable " + table, true);
+
+ // Should be about a 3 second scan
+ for (int i = 0; i < 6; i++) {
+ exec("insert " + i + " cf cq value", true);
+ }
exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
+ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
Thread thread = new Thread() {
public void run() {
try {
@@ -788,14 +831,30 @@ public class ShellServerTest {
}
};
thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
+
+ List<String> scans = new ArrayList<String>();
+ // Try to find the active scan for about 5seconds
+ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
+ String currentScans = exec("listscans", true);
+ String[] lines = currentScans.split("\n");
+ for (int scanOffset = 2; i < lines.length; i++) {
+ String currentScan = lines[scanOffset];
+ if (currentScan.contains(table)) {
+ scans.add(currentScan);
+ }
+ }
+ UtilWaitThread.sleep(100);
+ }
thread.join();
+
+ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
+
+ for (String scan : scans) {
+ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
+ String parts[] = scan.split("\\|");
+ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
+ }
+
exec("deletetable -f " + table, true);
}
@@ -873,13 +932,24 @@ public class ShellServerTest {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
+
+ private List<String> getFiles(String tableId) throws IOException {
+ output.clear();
- private int countFiles(String tableId) throws IOException {
exec("scan -t !METADATA -np -c file -b " + tableId + " -e " + tableId + "~");
-
- log.debug("countFiles(): " + output.get());
-
- return output.get().split("\n").length - 1;
+
+ String[] lines = StringUtils.split(output.get(), "\n");
+ output.clear();
+
+ if (0 == lines.length) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
+ }
+
+ private int countFiles(String tableId) throws IOException {
+ return getFiles(tableId).size();
}
private String getTableId(String tableName) throws Exception {
[02/16] git commit: ACCUMULO-2447 Always set PrintWriter on Shell and
don't print to stderr.
Posted by el...@apache.org.
ACCUMULO-2447 Always set PrintWriter on Shell and don't print to stderr.
While we don't have access to the printwriter/outputstream from the jline
ConsoleReader, we can force the user to pass in one (or just make one for
stdout). This will allow unit tests to correctly squash all output into
a bytearray (or similar) instead of having to redirect stderr too (which
would have other repercussions).
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e13c2733
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e13c2733
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e13c2733
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: e13c27331a42d55408427f18d97b285372f92a47
Parents: bc9cee0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:18:23 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +++++++-------------
.../accumulo/core/util/shell/ShellTest.java | 4 +++-
.../apache/accumulo/test/ShellServerTest.java | 3 ++-
3 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 656f2ae..6943b37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -202,16 +202,14 @@ public class Shell extends ShellOptions {
private boolean masking = false;
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
@@ -943,13 +941,9 @@ public class Shell extends ShellOptions {
}
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
public int getExitCode() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 02e9543..8505370 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
import jline.ConsoleReader;
@@ -81,7 +82,8 @@ public class ShellTest {
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index 28a4dd2..e2ebbcf 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -163,7 +163,8 @@ public class ShellServerTest {
public void setupShell() throws Exception {
// start the shell
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("-u", "root", "-p", secret, "-z", cluster.getInstanceName(), cluster.getZooKeepers());
exec("quit", true);
[03/16] git commit: ACCUMULO-2447 Always set PrintWriter on Shell and
don't print to stderr.
Posted by el...@apache.org.
ACCUMULO-2447 Always set PrintWriter on Shell and don't print to stderr.
While we don't have access to the printwriter/outputstream from the jline
ConsoleReader, we can force the user to pass in one (or just make one for
stdout). This will allow unit tests to correctly squash all output into
a bytearray (or similar) instead of having to redirect stderr too (which
would have other repercussions).
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e13c2733
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e13c2733
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e13c2733
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: e13c27331a42d55408427f18d97b285372f92a47
Parents: bc9cee0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:18:23 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +++++++-------------
.../accumulo/core/util/shell/ShellTest.java | 4 +++-
.../apache/accumulo/test/ShellServerTest.java | 3 ++-
3 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 656f2ae..6943b37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -202,16 +202,14 @@ public class Shell extends ShellOptions {
private boolean masking = false;
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
@@ -943,13 +941,9 @@ public class Shell extends ShellOptions {
}
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
public int getExitCode() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 02e9543..8505370 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
import jline.ConsoleReader;
@@ -81,7 +82,8 @@ public class ShellTest {
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index 28a4dd2..e2ebbcf 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -163,7 +163,8 @@ public class ShellServerTest {
public void setupShell() throws Exception {
// start the shell
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("-u", "root", "-p", secret, "-z", cluster.getInstanceName(), cluster.getZooKeepers());
exec("quit", true);
[10/16] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index 7a1121f,0000000..e0ad1bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@@ -1,1159 -1,0 +1,1238 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
++import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
++import java.util.ArrayList;
++import java.util.Arrays;
++import java.util.Collections;
++import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.commons.io.FileUtils;
++import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class ShellServerIT extends SimpleMacIT {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static final Logger log = Logger.getLogger(ShellServerIT.class);
+
+ public static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() throws IOException {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+
++ private static abstract class ErrorMessageCallback {
++ public abstract String getErrorMessage();
++ }
++
++ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
++ private static final String empty = "";
++ public String getErrorMessage() {
++ return empty;
++ }
++ }
++
++ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
++
+ public TestOutputStream output;
+ public StringInputStream input;
+ public Shell shell;
+ private static Process traceProcess;
+
+ @Rule
+ public TestName name = new TestName();
+
+ String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ String exec(String cmd, boolean expectGoodExit) throws IOException {
++ return exec(cmd, expectGoodExit, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
- assertGoodExit("", true);
++ assertGoodExit("", true, callback);
+ else
- assertBadExit("", true);
++ assertBadExit("", true, callback);
+ return result;
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
++ return exec(cmd, expectGoodExit, expectString, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
++ return exec(cmd, expectGoodExit, expectString, true, callback);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
++ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
++ assertGoodExit(expectString, stringPresent, callback);
+ else
- assertBadExit(expectString, stringPresent);
++ assertBadExit(expectString, stringPresent, callback);
+ return result;
+ }
+
+ void assertGoodExit(String s, boolean stringPresent) {
++ assertGoodExit(s, stringPresent, noop);
++ }
++
++ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.info(output.get());
- assertEquals(0, shell.getExitCode());
++ if (0 != shell.getExitCode()) {
++ String errorMsg = callback.getErrorMessage();
++ assertEquals(errorMsg, 0, shell.getExitCode());
++ }
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
- void assertBadExit(String s, boolean stringPresent) {
++ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.debug(output.get());
+ assertTrue(shell.getExitCode() > 0);
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // history file is updated in $HOME
+ System.setProperty("HOME", getFolder().getAbsolutePath());
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
+ // use reflection to call this method so it does not need to be made public
+ Method method = getStaticCluster().getClass().getDeclaredMethod("exec", Class.class, String[].class);
+ method.setAccessible(true);
+ traceProcess = (Process) method.invoke(getStaticCluster(), TraceServer.class, new String[0]);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @Before
+ public void setupShell() throws Exception {
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
- shell = new Shell(new ConsoleReader(input, output));
++ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
++ shell = new Shell(new ConsoleReader(input, output), pw);
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(),
+ "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ traceProcess.destroy();
+ }
+
+ @After
+ public void deleteTables() throws Exception {
+ Connector c = getConnector();
+ for (String table : c.tableOperations().list()) {
+ if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace"))
+ try {
+ c.tableOperations().delete(table);
+ } catch (TableNotFoundException e) {
+ // don't care
+ }
+ }
+ }
+
+ @After
+ public void tearDownShell() {
+ shell.shutdown();
+ }
+
+ @Test(timeout = 60000)
+ public void exporttableImporttable() throws Exception {
+ final String table = name.getMethodName(), table2 = table + "2";
+
+ // exporttable / importtable
+ exec("createtable " + table + " -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t " + table + " -s table.split.threshold=345M", true);
+ exec("offline " + table, true);
+ String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString();
+ exec("exporttable -t " + table + " " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable " + table2 + " " + import_, true);
+ exec("config -t " + table2 + " -np", true, "345M", true);
+ exec("getsplits -t " + table2, true, "row5", true);
+ exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true);
+ exec("onlinetable " + table, true);
+ exec("deletetable -f " + table, true);
+ exec("deletetable -f " + table2, true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
+ @Test(timeout = 45000)
+ public void setscaniterDeletescaniter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setscaniter, deletescaniter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING");
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder());
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
+ @Test(timeout = 45000)
+ public void egrep() throws Exception {
+ final String table = name.getMethodName();
+
+ // egrep
+ exec("createtable " + table);
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void du() throws Exception {
+ final String table = name.getMethodName();
+
+ // create and delete a table so we get out of a table context in the shell
+ exec("notable", true);
+
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
+
+ output.clear();
+ exec("createtable " + table);
+ make10();
+ exec("flush -t " + table + " -w");
+ exec("du " + table, true, " [" + table + "]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*[1-9][0-9][0-9]\\s\\[" + table + "\\]\\n"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
+ @Test(timeout = 45000)
+ public void user() throws Exception {
+ final String table = name.getMethodName();
+
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ input.set("secret\nsecret\n");
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (" + MetadataTable.NAME + "): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ input.set("secret\nsecret\n");
+ exec("user xyzzy", true);
+ exec("createtable " + table, true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f " + table, true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
+ @Test(timeout = 45000)
+ public void iter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n name", false);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f " + table);
+ // list iter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n name", false);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void setIterOptionPrompt() throws Exception {
+ Connector conn = getConnector();
+ String tableName = name.getMethodName();
+
+ exec("createtable " + tableName);
+ input.set("\n\n");
+ // Setting a non-optiondescriber with no name should fail
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", false);
+
+ // Name as option will work
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+
+ String expectedKey = "table.iterator.scan.cfcounter";
+ String expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ TableOperations tops = conn.tableOperations();
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\nname1 value1\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name1";
+ expectedValue = "value1";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("\nname1 value1.1,value1.2,value1.3\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+ expectedKey = "table.iterator.scan.cfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name1";
+ expectedValue = "value1.1,value1.2,value1.3";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ }
+
+ protected void checkTableForProperty(TableOperations tops, String tableName, String expectedKey, String expectedValue) throws Exception {
+ for (int i = 0; i < 5; i++) {
+ for (Entry<String,String> entry : tops.getProperties(tableName)) {
+ if (expectedKey.equals(entry.getKey())) {
+ assertEquals(expectedValue, entry.getValue());
+ return;
+ }
+ }
+ Thread.sleep(500);
+ }
+
+ fail("Failed to find expected property on " + tableName + ": " + expectedKey + "=" + expectedValue);
+ }
+
+ @Test(timeout = 45000)
+ public void notable() throws Exception {
+ final String table = name.getMethodName();
+
+ // notable
+ exec("createtable " + table, true);
+ exec("scan", true, " " + table + ">", true);
+ assertTrue(output.get().contains(" " + table + ">"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" " + table + ">"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue("Diff was actually " + diff, diff >= 200);
+ assertTrue("Diff was actually " + diff, diff < 600);
+ }
+
+ @Test(timeout = 45000)
+ public void addauths() throws Exception {
+ final String table = name.getMethodName();
+ // addauths
+ exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
++ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
++ public String getErrorMessage() {
++ try {
++ Connector c = getConnector();
++ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
++ } catch (Exception e) {
++ return "Could not check authorizations";
++ }
++ }
++ });
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
+ @Test(timeout = 45000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ if (shell.getReader().getTerminal().isAnsiSupported()) {
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ } else {
+ exec("cls", false, "does not support");
+ exec("clear", false, "does not support");
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void clonetable() throws Exception {
+ final String table = name.getMethodName(), clone = table + "_clone";
+
+ // clonetable
+ exec("createtable " + table + " -evc");
+ exec("config -t " + table + " -s table.split.threshold=123M", true);
+ exec("addsplits -t " + table + " a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable " + table + " " + clone);
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t " + clone, true, "VisibilityConstraint=2", true);
+ exec("config -t " + clone + " -np", true, "123M", true);
+ exec("getsplits -t " + clone, true, "a\nb\nc\n");
+ exec("deletetable -f " + table);
+ exec("deletetable -f " + clone);
+ }
+
+ @Test(timeout = 45000)
+ public void testCompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ // compact
+ exec("createtable " + table);
+
+ String tableId = getTableId(table);
+
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles(tableId);
+ // merge two files into one
+ exec("compact -t " + table + " -w");
+ assertTrue(countFiles(tableId) < oldCount);
+ exec("addsplits -t " + table + " f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
- oldCount = countFiles(tableId);
++ List<String> oldFiles = getFiles(tableId);
+
+ // at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
++ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
+
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertEquals(2, countFiles(tableId));
+ exec("compact -w");
+ assertEquals(2, countFiles(tableId));
+ exec("merge --all -t " + table);
+ exec("compact -w");
+ assertEquals(1, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void constraint() throws Exception {
+ final String table = name.getMethodName();
+
+ // constraint
+ exec("constraint -l -t " + MetadataTable.NAME + "", true, "MetadataConstraints=1", true);
+ exec("createtable " + table + " -evc");
+
+ // Make sure the table is fully propagated through zoocache
+ getTableId(table);
+
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", true);
+ exec("constraint -t " + table + " -d 2", true, "Removed constraint 2 from table " + table);
+ // wait for zookeeper updates to propagate
+ UtilWaitThread.sleep(1000);
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deletemany() throws Exception {
+ final String table = name.getMethodName();
+
+ // deletemany
+ exec("createtable " + table);
+ make10();
+ assertEquals(10, countkeys(table));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys(table));
+ exec("scan -t " + table + " -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys(table));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deleterows() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ final String tableId = getTableId(table);
+
+ // deleterows
+ int base = countFiles(tableId);
+ assertEquals(0, base);
+
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
++ List<String> files = getFiles(tableId);
++ assertEquals("Found the following files: " + files, 3, files.size());
+ exec("deleterows -t " + table + " -b row5 -e row7", true);
+ assertEquals(2, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void groups() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("setgroups -t " + table + " alpha=a,b,c num=3,2,1");
+ exec("getgroups -t " + table, true, "alpha=a,b,c", true);
+ exec("getgroups -t " + table, true, "num=1,2,3", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void grep() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
+
+ // @Test(timeout = 45000)
+ public void history() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("history -c", true);
+ exec("createtable " + table);
+ exec("deletetable -f " + table);
+ exec("history", true, table, true);
+ exec("history", true, "history", true);
+ }
+
+ @Test(timeout = 45000)
+ public void importDirectory() throws Exception {
+ final String table = name.getMethodName();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = new File(getFolder(), "import");
+ importDir.mkdir();
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = new File(getFolder(), "errors");
+ errorsDir.mkdir();
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ assertEquals(0, shell.getExitCode());
+ exec("createtable " + table, true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
+ @Test(timeout = 45000)
+ public void interpreter() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+ // Need to allow time for this to propagate through zoocache/zookeeper
+ UtilWaitThread.sleep(3000);
+
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void listcompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.minc.slow.opt.sleepTime=1000", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t " + table, true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void maxrow() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void merge() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f " + table);
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(2, output.get().split("\n").length);
+ exec("getsplits -t accumulo.root", true);
+ assertEquals(1, output.get().split("\n").length);
+ exec("merge --all -t " + MetadataTable.NAME + "");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(1, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void renametable() throws Exception {
+ final String table = name.getMethodName() + "1", rename = name.getMethodName() + "2";
+
+ exec("createtable " + table);
+ exec("insert this is a value");
+ exec("renametable " + table + " " + rename);
+ exec("tables", true, rename, true);
+ exec("tables", true, table, false);
+ exec("scan -t " + rename, true, "value", true);
+ exec("deletetable -f " + rename, true);
+ }
+
+ @Test(timeout = 30000)
+ public void tables() throws Exception {
+ final String table = name.getMethodName(), table1 = table + "_z", table2 = table + "_a";
+ exec("createtable " + table1);
+ exec("createtable " + table2);
+ exec("notable");
+ String lst = exec("tables -l");
+ assertTrue(lst.indexOf(table2) < lst.indexOf(table1));
+ lst = exec("tables -l -s");
+ assertTrue(lst.indexOf(table1) < lst.indexOf(table2));
+ }
+
+ @Test(timeout = 45000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(11, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
+ @Test(timeout = 45000)
+ public void listscans() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
++
++ // Should be about a 3 second scan
++ for (int i = 0; i < 6; i++) {
++ exec("insert " + i + " cf cq value", true);
++ }
+ exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
++ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Connector connector = getConnector();
+ Scanner s = connector.createScanner(table, Authorizations.EMPTY);
+ FunctionalTestUtils.count(s);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
- String hostPortPattern = ".+:\\d+";
- String tserver = parts[0].trim();
- assertTrue(tserver.matches(hostPortPattern));
- assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
- String client = parts[1].trim();
- assertTrue(client.matches(hostPortPattern));
- // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
++
++ List<String> scans = new ArrayList<String>();
++ // Try to find the active scan for about 5seconds
++ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
++ String currentScans = exec("listscans", true);
++ String[] lines = currentScans.split("\n");
++ for (int scanOffset = 2; i < lines.length; i++) {
++ String currentScan = lines[scanOffset];
++ if (currentScan.contains(table)) {
++ scans.add(currentScan);
++ }
++ }
++ UtilWaitThread.sleep(100);
++ }
+ thread.join();
++
++ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
++
++ for (String scan : scans) {
++ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
++ String parts[] = scan.split("\\|");
++ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
++ String tserver = parts[0].trim();
++ // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
++ String hostPortPattern = ".+:\\d+";
++ assertTrue(tserver.matches(hostPortPattern));
++ assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
++ String client = parts[1].trim();
++ assertTrue(client.matches(hostPortPattern));
++ }
++
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void testPertableClasspath() throws Exception {
+ final String table = name.getMethodName();
+
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar", getFolder());
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t " + table + " -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline -w " + table);
+ exec("online -w " + table);
+
+ exec("table " + table, true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f " + table, true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
+ @Test(timeout = 45000)
+ public void trace() throws Exception {
+ // Make sure to not collide with the "trace" table
+ final String table = name.getMethodName() + "Test";
+
+ exec("trace on", true);
+ exec("createtable " + table, true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f " + table);
+ exec("sleep 1");
+ String trace = exec("trace off");
+ System.out.println(trace);
+ assertTrue(trace.contains("sendMutations"));
+ assertTrue(trace.contains("startScan"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
+ @Test(timeout = 30000)
+ public void badLogin() throws Exception {
+ input.set(ROOT_PASSWORD + "\n");
+ String err = exec("user NoSuchUser", false);
+ assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser"));
+ }
+
+ @Test(timeout = 30000)
+ public void namespaces() throws Exception {
+ exec("namespaces", true, "\"\"", true); // default namespace, displayed as quoted empty string
+ exec("namespaces", true, Namespaces.ACCUMULO_NAMESPACE, true);
+ exec("createnamespace thing1", true);
+ String namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing1"));
+
+ exec("renamenamespace thing1 thing2");
+ namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing2"));
+ assertTrue(!namespaces.contains("thing1"));
+
+ // can't delete a namespace that still contains tables, unless you do -f
+ exec("createtable thing2.thingy", true);
+ exec("deletenamespace thing2");
+ exec("y");
+ exec("namespaces", true, "thing2", true);
+
+ exec("du -ns thing2", true, "thing2.thingy", true);
+
+ // all "TableOperation" commands can take a namespace
+ exec("offline -ns thing2", true);
+ exec("online -ns thing2", true);
+ exec("flush -ns thing2", true);
+ exec("compact -ns thing2", true);
+ exec("createnamespace testers3", true);
+ exec("createtable testers3.1", true);
+ exec("createtable testers3.2", true);
+ exec("deletetable -ns testers3 -f", true);
+ exec("tables", true, "testers3.1", false);
+ exec("namespaces", true, "testers3", true);
+ exec("deletenamespace testers3 -f", true);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -ns thing2 -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("listiter -ns thing2 -scan", true, "Summing", true);
+ exec("deleteiter -ns thing2 -n name -scan", true);
+ exec("createuser dude");
+ exec("pass");
+ exec("pass");
+ exec("grant Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+ exec("revoke Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+
+ // properties override and such
+ exec("config -ns thing2 -s table.file.max=44444", true);
+ exec("config -ns thing2", true, "44444", true);
+ exec("config -t thing2.thingy", true, "44444", true);
+ exec("config -t thing2.thingy -s table.file.max=55555", true);
+ exec("config -t thing2.thingy", true, "55555", true);
+
+ // can copy properties when creating
+ exec("createnamespace thing3 -cc thing2", true);
+ exec("config -ns thing3", true, "44444", true);
+
+ exec("deletenamespace -f thing2", true);
+ exec("namespaces", true, "thing2", false);
+ exec("tables", true, "thing2.thingy", false);
+
+ // put constraints on a namespace
+ exec("constraint -ns thing3 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ exec("createtable thing3.constrained", true);
+ exec("table thing3.constrained", true);
+ exec("constraint -d 1");
+ // should fail
+ exec("constraint -l", true, "NumericValueConstraint", true);
+ exec("insert r cf cq abc", false);
+ exec("constraint -ns thing3 -d 1");
+ exec("sleep 1");
+ exec("insert r cf cq abc", true);
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
+ @Test(timeout = 30000)
+ public void scans() throws Exception {
+ exec("createtable t");
+ make10();
+ String result = exec("scan -np -b row1 -e row1");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row3 -e row5");
+ assertEquals(4, result.split("\n").length);
+ result = exec("scan -np -r row3");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row:");
+ assertEquals(1, result.split("\n").length);
+ result = exec("scan -np -b row");
+ assertEquals(11, result.split("\n").length);
+ result = exec("scan -np -e row:");
+ assertEquals(11, result.split("\n").length);
+ exec("deletetable -f t");
+ }
+
+ @Test(timeout = 30000)
+ public void whoami() throws Exception {
+ assertTrue(exec("whoami", true).contains("root"));
+ input.set("secret\nsecret\n");
+ exec("createuser test_user");
+ exec("setauths -u test_user -s 12,3,4");
+ String auths = exec("getauths -u test_user");
+ assertTrue(auths.contains("3") && auths.contains("12") && auths.contains("4"));
+ input.set("secret\n");
+ exec("user test_user", true);
+ assertTrue(exec("whoami", true).contains("test_user"));
+ input.set(ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
++
++ private List<String> getFiles(String tableId) throws IOException {
++ output.clear();
+
- private int countFiles(String tableId) throws IOException {
+ exec("scan -t " + MetadataTable.NAME + " -np -c file -b " + tableId + " -e " + tableId + "~");
+
+ log.debug("countFiles(): " + output.get());
+
- return output.get().split("\n").length - 1;
++ String[] lines = StringUtils.split(output.get(), "\n");
++ output.clear();
++
++ if (0 == lines.length) {
++ return Collections.emptyList();
++ }
++
++ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
++ }
++
++ private int countFiles(String tableId) throws IOException {
++ return getFiles(tableId).size();
+ }
+
+ private String getTableId(String tableName) throws Exception {
+ Connector conn = getConnector();
+
+ for (int i = 0; i < 5; i++) {
+ Map<String,String> nameToId = conn.tableOperations().tableIdMap();
+ if (nameToId.containsKey(tableName)) {
+ return nameToId.get(tableName);
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+
+ fail("Could not find ID for table: " + tableName);
+ // Will never get here
+ return null;
+ }
+
+}
[09/16] git commit: ACCUMULO-2433 Add a way to still use the exec
methods but provide a custom error message on failure.
Posted by el...@apache.org.
ACCUMULO-2433 Add a way to still use the exec methods but provide a custom error message on failure.
Fix some tests that I've seen fail to provide better messages on failure to help future debugging
because I can't get them reproduced locally in any reasonable timeframe.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4cc25caa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4cc25caa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4cc25caa
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: 4cc25caa07ea5ca49e4d81f9e5a22908e45bba59
Parents: 72395e0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:59:09 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 17:13:22 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/test/ShellServerTest.java | 128 ++++++++++++++-----
1 file changed, 99 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4cc25caa/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index e2ebbcf..d66f509 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -31,6 +31,10 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -53,6 +57,7 @@ import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.server.trace.TraceServer;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -85,7 +90,19 @@ public class ShellServerTest {
}
}
+ private static abstract class ErrorMessageCallback {
+ public abstract String getErrorMessage();
+ }
+
+ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
+ private static final String empty = "";
+ public String getErrorMessage() {
+ return empty;
+ }
+ }
+
private static final Logger log = Logger.getLogger(ShellServerTest.class);
+ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
@@ -104,36 +121,51 @@ public class ShellServerTest {
}
String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit("", true);
+ assertGoodExit("", true, callback);
else
- assertBadExit("", true);
+ assertBadExit("", true, callback);
return result;
}
String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
}
String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
+ assertGoodExit(expectString, stringPresent, callback);
else
- assertBadExit(expectString, stringPresent);
+ assertBadExit(expectString, stringPresent, callback);
return result;
}
- void assertGoodExit(String s, boolean stringPresent) {
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
- assertEquals(0, shell.getExitCode());
+ if (0 != shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(0, shell.getExitCode());
+ }
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
- void assertBadExit(String s, boolean stringPresent) {
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
if (s.length() > 0)
@@ -409,7 +441,16 @@ public class ShellServerTest {
final String table = name.getMethodName();
// addauths
exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
+ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
+ public String getErrorMessage() {
+ try {
+ Connector c = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getConnector("root", new PasswordToken(secret));
+ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+ });
exec("addauths -s foo,bar", true);
exec("getauths", true, "foo", true);
exec("getauths", true, "bar", true);
@@ -486,10 +527,10 @@ public class ShellServerTest {
exec("flush -w");
exec("insert n 1 2 3");
exec("flush -w");
- oldCount = countFiles(tableId);
+ List<String> oldFiles = getFiles(tableId);
// at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
+ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
// compact some data:
exec("compact -b g -e z -w");
@@ -564,7 +605,8 @@ public class ShellServerTest {
exec("addsplits row5 row7");
make10();
exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
+ List<String> files = getFiles(tableId);
+ assertEquals("Found the following files: " + files, 3, files.size());
exec("deleterows -t " + table + " -b row5 -e row7", true);
assertEquals(2, countFiles(tableId));
exec("deletetable -f " + table);
@@ -767,12 +809,13 @@ public class ShellServerTest {
final String table = name.getMethodName();
exec("createtable " + table, true);
+
+ // Should be about a 3 second scan
+ for (int i = 0; i < 6; i++) {
+ exec("insert " + i + " cf cq value", true);
+ }
exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
+ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
Thread thread = new Thread() {
public void run() {
try {
@@ -788,14 +831,30 @@ public class ShellServerTest {
}
};
thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
+
+ List<String> scans = new ArrayList<String>();
+ // Try to find the active scan for about 5seconds
+ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
+ String currentScans = exec("listscans", true);
+ String[] lines = currentScans.split("\n");
+ for (int scanOffset = 2; i < lines.length; i++) {
+ String currentScan = lines[scanOffset];
+ if (currentScan.contains(table)) {
+ scans.add(currentScan);
+ }
+ }
+ UtilWaitThread.sleep(100);
+ }
thread.join();
+
+ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
+
+ for (String scan : scans) {
+ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
+ String parts[] = scan.split("\\|");
+ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
+ }
+
exec("deletetable -f " + table, true);
}
@@ -873,13 +932,24 @@ public class ShellServerTest {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
+
+ private List<String> getFiles(String tableId) throws IOException {
+ output.clear();
- private int countFiles(String tableId) throws IOException {
exec("scan -t !METADATA -np -c file -b " + tableId + " -e " + tableId + "~");
-
- log.debug("countFiles(): " + output.get());
-
- return output.get().split("\n").length - 1;
+
+ String[] lines = StringUtils.split(output.get(), "\n");
+ output.clear();
+
+ if (0 == lines.length) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
+ }
+
+ private int countFiles(String tableId) throws IOException {
+ return getFiles(tableId).size();
}
private String getTableId(String tableName) throws Exception {
[16/16] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/389003fd
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/389003fd
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/389003fd
Branch: refs/heads/master
Commit: 389003fdeeffbce540c4ac8c1d025c774e6e5eeb
Parents: 977d41d da7f937
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 18:15:36 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 18:15:36 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +--
.../core/util/shell/ShellConfigTest.java | 5 +-
.../core/util/shell/ShellSetInstanceTest.java | 3 +-
.../accumulo/core/util/shell/ShellTest.java | 5 +-
.../apache/accumulo/proxy/SimpleProxyIT.java | 98 +++++++++----
.../org/apache/accumulo/test/ShellServerIT.java | 145 ++++++++++++++-----
6 files changed, 196 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
[14/16] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --cc proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index abe540a,0000000..efe36b9
mode 100644,000000..100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@@ -1,1529 -1,0 +1,1567 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.proxy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.DevNull;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
+import org.apache.accumulo.proxy.thrift.AccumuloSecurityException;
+import org.apache.accumulo.proxy.thrift.ActiveCompaction;
+import org.apache.accumulo.proxy.thrift.ActiveScan;
+import org.apache.accumulo.proxy.thrift.BatchScanOptions;
+import org.apache.accumulo.proxy.thrift.Column;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.CompactionReason;
+import org.apache.accumulo.proxy.thrift.CompactionType;
+import org.apache.accumulo.proxy.thrift.Condition;
+import org.apache.accumulo.proxy.thrift.ConditionalStatus;
+import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
+import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
+import org.apache.accumulo.proxy.thrift.DiskUsage;
+import org.apache.accumulo.proxy.thrift.IteratorScope;
+import org.apache.accumulo.proxy.thrift.IteratorSetting;
+import org.apache.accumulo.proxy.thrift.Key;
+import org.apache.accumulo.proxy.thrift.KeyValue;
+import org.apache.accumulo.proxy.thrift.MutationsRejectedException;
+import org.apache.accumulo.proxy.thrift.PartialKey;
+import org.apache.accumulo.proxy.thrift.Range;
+import org.apache.accumulo.proxy.thrift.ScanColumn;
+import org.apache.accumulo.proxy.thrift.ScanOptions;
+import org.apache.accumulo.proxy.thrift.ScanResult;
+import org.apache.accumulo.proxy.thrift.ScanState;
+import org.apache.accumulo.proxy.thrift.ScanType;
+import org.apache.accumulo.proxy.thrift.SystemPermission;
+import org.apache.accumulo.proxy.thrift.TableExistsException;
+import org.apache.accumulo.proxy.thrift.TableNotFoundException;
+import org.apache.accumulo.proxy.thrift.TablePermission;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.accumulo.proxy.thrift.UnknownScanner;
+import org.apache.accumulo.proxy.thrift.UnknownWriter;
+import org.apache.accumulo.proxy.thrift.WriterOptions;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.test.functional.SlowIterator;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.server.TServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+/**
+ * Call every method on the proxy and try to verify that it works.
+ */
+public class SimpleProxyIT {
+
+ public static File macTestFolder = new File(System.getProperty("user.dir") + "/target/" + SimpleProxyIT.class.getName());
+
+ private static MiniAccumuloCluster accumulo;
+ private static String secret = "superSecret";
+ private static Random random = new Random();
+ private static TServer proxyServer;
+ private static Thread thread;
+ private static int proxyPort;
+ private static org.apache.accumulo.proxy.thrift.AccumuloProxy.Client client;
+ private static String principal = "root";
+
+ private static Map<String,String> properties = new TreeMap<String,String>() {
+ private static final long serialVersionUID = 1L;
+
+ {
+ put("password", secret);
+ }
+ };
+ private static ByteBuffer creds = null;
+
+ private static Class<? extends TProtocolFactory> protocolClass;
+
+ static Class<? extends TProtocolFactory> getRandomProtocol() {
+ List<Class<? extends TProtocolFactory>> protocolFactories = new ArrayList<Class<? extends TProtocolFactory>>();
+ protocolFactories.add(org.apache.thrift.protocol.TJSONProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TBinaryProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TTupleProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TCompactProtocol.Factory.class);
+
+ return protocolFactories.get(random.nextInt(protocolFactories.size()));
+ }
+
+ private static final AtomicInteger tableCounter = new AtomicInteger(0);
+
+ private static String makeTableName() {
+ return "test" + tableCounter.getAndIncrement();
+ }
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(macTestFolder);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+ FileUtils.deleteQuietly(macTestFolder);
+ macTestFolder.mkdirs();
+ MiniAccumuloConfig config = new MiniAccumuloConfig(macTestFolder, secret).setNumTservers(1);
+ accumulo = new MiniAccumuloCluster(config);
+ accumulo.start();
+
+ Properties props = new Properties();
+ props.put("instance", accumulo.getConfig().getInstanceName());
+ props.put("zookeepers", accumulo.getZooKeepers());
+ props.put("tokenClass", PasswordToken.class.getName());
+
+ protocolClass = getRandomProtocol();
+
+ proxyPort = PortUtils.getRandomFreePort();
+ proxyServer = Proxy.createProxyServer(org.apache.accumulo.proxy.thrift.AccumuloProxy.class, org.apache.accumulo.proxy.ProxyServer.class, proxyPort,
+ protocolClass, props);
+ thread = new Thread() {
+ @Override
+ public void run() {
+ proxyServer.serve();
+ }
+ };
+ thread.start();
+ while (!proxyServer.isServing())
+ UtilWaitThread.sleep(100);
+ client = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ creds = client.login(principal, properties);
+ }
+
+ @Test(timeout = 10000)
+ public void security() throws Exception {
+ client.createLocalUser(creds, "user", s2bb(secret));
+ ByteBuffer badLogin = client.login("user", properties);
+ client.dropLocalUser(creds, "user");
+ final String table = makeTableName();
+ client.createTable(creds, table, false, TimeType.MILLIS);
+
+ final IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ try {
+ client.addConstraint(badLogin, table, NumericValueConstraint.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.addSplits(badLogin, table, Collections.singleton(s2bb("1")));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.clearLocatorCache(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.compactTable(badLogin, table, null, null, null, true, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.cancelCompaction(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createTable(badLogin, table, false, TimeType.MILLIS);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.deleteTable(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.deleteRows(badLogin, table, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.tableExists(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.flushTable(badLogin, table, null, null, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getLocalityGroups(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getMaxRow(badLogin, table, Collections.<ByteBuffer> emptySet(), null, false, null, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getTableProperties(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listSplits(badLogin, table, 10000);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listTables(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.listConstraints(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.mergeTablets(badLogin, table, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.offlineTable(badLogin, table, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.onlineTable(badLogin, table, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeConstraint(badLogin, table, 0);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeTableProperty(badLogin, table, Property.TABLE_FILE_MAX.getKey());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.renameTable(badLogin, table, "someTableName");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ Map<String,Set<String>> groups = new HashMap<String,Set<String>>();
+ groups.put("group1", Collections.singleton("cf1"));
+ groups.put("group2", Collections.singleton("cf2"));
+ client.setLocalityGroups(badLogin, table, groups);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.setTableProperty(badLogin, table, Property.TABLE_FILE_MAX.getKey(), "0");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.tableIdMap(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.getSiteConfiguration(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getSystemConfiguration(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getTabletServers(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.getActiveScans(badLogin, "fake");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getActiveCompactions(badLogin, "fakse");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeProperty(badLogin, "table.split.threshold");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.setProperty(badLogin, "table.split.threshold", "500M");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.testClassLoad(badLogin, DevNull.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.authenticateUser(badLogin, "root", s2pp(secret));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ HashSet<ByteBuffer> auths = new HashSet<ByteBuffer>(Arrays.asList(s2bb("A"), s2bb("B")));
+ client.changeUserAuthorizations(badLogin, "stooge", auths);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.changeLocalUserPassword(badLogin, "stooge", s2bb(""));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createLocalUser(badLogin, "stooge", s2bb("password"));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.dropLocalUser(badLogin, "stooge");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getUserAuthorizations(badLogin, "stooge");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.grantSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.grantTablePermission(badLogin, "root", table, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.hasSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.hasTablePermission(badLogin, "root", table, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listLocalUsers(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.revokeSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.revokeTablePermission(badLogin, "root", table, TablePermission.ALTER_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createScanner(badLogin, table, new ScanOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createBatchScanner(badLogin, table, new BatchScanOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.updateAndFlush(badLogin, table, new HashMap<ByteBuffer,List<ColumnUpdate>>());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createWriter(badLogin, table, new WriterOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.attachIterator(badLogin, "slow", setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.checkIteratorConflicts(badLogin, table, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ final String TABLE_TEST = makeTableName();
+ client.cloneTable(badLogin, table, TABLE_TEST, false, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.exportTable(badLogin, table, "/tmp");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.importTable(badLogin, "testify", "/tmp");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getIteratorSetting(badLogin, table, "foo", IteratorScope.SCAN);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listIterators(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeIterator(badLogin, table, "name", EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.splitRangeByTablets(badLogin, table, client.getRowRange(ByteBuffer.wrap("row".getBytes())), 10);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ File importDir = tempFolder.newFolder("importDir");
+ File failuresDir = tempFolder.newFolder("failuresDir");
+ client.importDirectory(badLogin, table, importDir.getAbsolutePath(), failuresDir.getAbsolutePath(), true);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.pingTabletServer(badLogin, "fake");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.login("badUser", properties);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.testTableClassLoad(badLogin, table, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createConditionalWriter(badLogin, table, new ConditionalWriterOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ }
+
+ @Test(timeout = 10000)
+ public void tableNotFound() throws Exception {
+ final String doesNotExist = "doesNotExists";
+ try {
+ client.addConstraint(creds, doesNotExist, NumericValueConstraint.class.getName());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.addSplits(creds, doesNotExist, Collections.<ByteBuffer> emptySet());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ final IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+ try {
+ client.attachIterator(creds, doesNotExist, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.cancelCompaction(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.checkIteratorConflicts(creds, doesNotExist, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.clearLocatorCache(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ final String TABLE_TEST = makeTableName();
+ client.cloneTable(creds, doesNotExist, TABLE_TEST, false, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.compactTable(creds, doesNotExist, null, null, null, true, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createBatchScanner(creds, doesNotExist, new BatchScanOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createScanner(creds, doesNotExist, new ScanOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createWriter(creds, doesNotExist, new WriterOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.deleteRows(creds, doesNotExist, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.deleteTable(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.exportTable(creds, doesNotExist, "/tmp");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.flushTable(creds, doesNotExist, null, null, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getIteratorSetting(creds, doesNotExist, "foo", IteratorScope.SCAN);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getLocalityGroups(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getMaxRow(creds, doesNotExist, Collections.<ByteBuffer> emptySet(), null, false, null, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getTableProperties(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.grantTablePermission(creds, "root", doesNotExist, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.hasTablePermission(creds, "root", doesNotExist, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ File importDir = tempFolder.newFolder("importDir");
+ File failuresDir = tempFolder.newFolder("failuresDir");
+ client.importDirectory(creds, doesNotExist, importDir.getAbsolutePath(), failuresDir.getAbsolutePath(), true);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.listConstraints(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.listSplits(creds, doesNotExist, 10000);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.mergeTablets(creds, doesNotExist, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.offlineTable(creds, doesNotExist, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.onlineTable(creds, doesNotExist, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeConstraint(creds, doesNotExist, 0);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeIterator(creds, doesNotExist, "name", EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeTableProperty(creds, doesNotExist, Property.TABLE_FILE_MAX.getKey());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.renameTable(creds, doesNotExist, "someTableName");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.revokeTablePermission(creds, "root", doesNotExist, TablePermission.ALTER_TABLE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.setTableProperty(creds, doesNotExist, Property.TABLE_FILE_MAX.getKey(), "0");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.splitRangeByTablets(creds, doesNotExist, client.getRowRange(ByteBuffer.wrap("row".getBytes())), 10);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.updateAndFlush(creds, doesNotExist, new HashMap<ByteBuffer,List<ColumnUpdate>>());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getDiskUsage(creds, Collections.singleton(doesNotExist));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.testTableClassLoad(creds, doesNotExist, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createConditionalWriter(creds, doesNotExist, new ConditionalWriterOptions());
+ } catch (TableNotFoundException ex) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testExists() throws Exception {
+ client.createTable(creds, "ett1", false, TimeType.MILLIS);
+ client.createTable(creds, "ett2", false, TimeType.MILLIS);
+ try {
+ client.createTable(creds, "ett1", false, TimeType.MILLIS);
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ try {
+ client.renameTable(creds, "ett1", "ett2");
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ try {
+ client.cloneTable(creds, "ett1", "ett2", false, new HashMap<String,String>(), new HashSet<String>());
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testUnknownScanner() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ String scanner = client.createScanner(creds, TABLE_TEST, null);
+ assertFalse(client.hasNext(scanner));
+ client.closeScanner(scanner);
+
+ try {
+ client.hasNext(scanner);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+
+ try {
+ client.closeScanner(scanner);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+
+ try {
+ client.nextEntry("99999999");
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.nextK("99999999", 6);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.hasNext("99999999");
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.hasNext(UUID.randomUUID().toString());
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testUnknownWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ String writer = client.createWriter(creds, TABLE_TEST, null);
+ client.update(writer, mutation("row0", "cf", "cq", "value"));
+ client.flush(writer);
+ client.update(writer, mutation("row2", "cf", "cq", "value2"));
+ client.closeWriter(writer);
+
+ // this is a oneway call, so it does not throw exceptions
+ client.update(writer, mutation("row2", "cf", "cq", "value2"));
+
+ try {
+ client.flush(writer);
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.flush("99999");
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.flush(UUID.randomUUID().toString());
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.closeWriter("99999");
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testDelete() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row0", "cf", "cq", "value"));
+
+ assertScan(new String[][] {{"row0", "cf", "cq", "value"}}, TABLE_TEST);
+
+ ColumnUpdate upd = new ColumnUpdate(s2bb("cf"), s2bb("cq"));
+ upd.setDeleteCell(true);
+ Map<ByteBuffer,List<ColumnUpdate>> delete = Collections.singletonMap(s2bb("row0"), Collections.singletonList(upd));
+
+ client.updateAndFlush(creds, TABLE_TEST, delete);
+
+ assertScan(new String[][] {}, TABLE_TEST);
+ }
+
- @Test(timeout = 10000)
++ @Test(timeout = 60000)
+ public void testInstanceOperations() throws Exception {
+ int tservers = 0;
+ for (String tserver : client.getTabletServers(creds)) {
+ client.pingTabletServer(creds, tserver);
+ tservers++;
+ }
+ assertTrue(tservers > 0);
+
+ // get something we know is in the site config
+ Map<String,String> cfg = client.getSiteConfiguration(creds);
+ assertTrue(cfg.get("instance.dfs.dir").startsWith(macTestFolder.getPath()));
+
+ // set a property in zookeeper
+ client.setProperty(creds, "table.split.threshold", "500M");
+
+ // check that we can read it
+ for (int i = 0; i < 5; i++) {
+ cfg = client.getSystemConfiguration(creds);
+ if ("500M".equals(cfg.get("table.split.threshold")))
+ break;
+ UtilWaitThread.sleep(200);
+ }
+ assertEquals("500M", cfg.get("table.split.threshold"));
+
+ // unset the setting, check that it's not what it was
+ client.removeProperty(creds, "table.split.threshold");
+ for (int i = 0; i < 5; i++) {
+ cfg = client.getSystemConfiguration(creds);
+ if (!"500M".equals(cfg.get("table.split.threshold")))
+ break;
+ UtilWaitThread.sleep(200);
+ }
+ assertNotEquals("500M", cfg.get("table.split.threshold"));
+
+ // try to load some classes via the proxy
+ assertTrue(client.testClassLoad(creds, DevNull.class.getName(), SortedKeyValueIterator.class.getName()));
+ assertFalse(client.testClassLoad(creds, "foo.bar", SortedKeyValueIterator.class.getName()));
+
+ // create a table that's very slow, so we can look for scans/compactions
+ client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
++
++ // Should take 5 seconds to read every record
++ for (int i = 0; i < 20; i++) {
++ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
++ }
++
++ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
+ client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
+
+ // scan
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ String scanner;
+ try {
+ Client client2 = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ scanner = client2.createScanner(creds, "slow", null);
+ client2.nextK(scanner, 10);
+ client2.closeScanner(scanner);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ t.start();
++
+ // look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
++ List<ActiveScan> scans = new ArrayList<ActiveScan>();
++ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
+ for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
++ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
++ for (ActiveScan scan : scansForServer) {
++ if ("root".equals(scan.getUser())) {
++ scans.add(scan);
++ }
++ }
++
+ if (!scans.isEmpty())
- break loop;
++ break;
+ UtilWaitThread.sleep(10);
+ }
+ }
+ t.join();
++
+ assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
++ boolean found = false;
++ Map<String,String> map = null;
++ for (int i = 0; i < scans.size() && !found; i++) {
++ ActiveScan scan = scans.get(i);
++ if ("root".equals(scan.getUser())) {
++ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
++ assertEquals(ScanType.SINGLE, scan.getType());
++ assertEquals("slow", scan.getTable());
++
++ map = client.tableIdMap(creds);
++ assertEquals(map.get("slow"), scan.getExtent().tableId);
++ assertTrue(scan.getExtent().endRow == null);
++ assertTrue(scan.getExtent().prevEndRow == null);
++ found = true;
++ }
++ }
++
++ assertTrue("Could not find a scan against the 'slow' table", found);
+
+ // start a compaction
+ t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Client client2 = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ client2.compactTable(creds, "slow", null, null, null, true, true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ t.start();
+
++ final String desiredTableId = map.get("slow");
++
+ // try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
++ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
++ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
++ // Iterate over the tservers
+ for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
++ // And get the compactions on each
++ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
++ for (ActiveCompaction compact : compactionsOnServer) {
++ // There might be other compactions occurring (e.g. on METADATA) in which
++ // case we want to prune out those that aren't for our slow table
++ if (desiredTableId.equals(compact.getExtent().tableId)) {
++ compactions.add(compact);
++ }
++ }
++
++ // If we found a compaction for the table we wanted, so we can stop looking
+ if (!compactions.isEmpty())
- break loop2;
++ break;
+ }
+ UtilWaitThread.sleep(10);
+ }
+ t.join();
++
+ // verify the compaction information
+ assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
++ for (ActiveCompaction c : compactions) {
++ if (desiredTableId.equals(c.getExtent().tableId)) {
++ assertTrue(c.inputFiles.isEmpty());
++ assertEquals(CompactionType.MINOR, c.getType());
++ assertEquals(CompactionReason.USER, c.getReason());
++ assertEquals("", c.localityGroup);
++ assertTrue(c.outputFile.contains("default_tablet"));
++
++ return;
++ }
++ }
++ fail("Expection to find running compaction for table 'slow' but did not find one");
+ }
+
+ @Test
+ public void testSecurityOperations() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ // check password
+ assertTrue(client.authenticateUser(creds, "root", s2pp(secret)));
+ assertFalse(client.authenticateUser(creds, "root", s2pp("")));
+
+ // create a user
+ client.createLocalUser(creds, "stooge", s2bb("password"));
+ // change auths
+ Set<String> users = client.listLocalUsers(creds);
+ assertEquals(new HashSet<String>(Arrays.asList("root", "stooge")), users);
+ HashSet<ByteBuffer> auths = new HashSet<ByteBuffer>(Arrays.asList(s2bb("A"), s2bb("B")));
+ client.changeUserAuthorizations(creds, "stooge", auths);
+ List<ByteBuffer> update = client.getUserAuthorizations(creds, "stooge");
+ assertEquals(auths, new HashSet<ByteBuffer>(update));
+
+ // change password
+ client.changeLocalUserPassword(creds, "stooge", s2bb(""));
+ assertTrue(client.authenticateUser(creds, "stooge", s2pp("")));
+
+ // check permission failure
+ @SuppressWarnings("serial")
+ ByteBuffer stooge = client.login("stooge", new TreeMap<String,String>() {
+ {
+ put("password", "");
+ }
+ });
+
+ try {
+ client.createTable(stooge, "fail", true, TimeType.MILLIS);
+ fail("should not create the table");
+ } catch (AccumuloSecurityException ex) {
+ assertFalse(client.listTables(creds).contains("fail"));
+ }
+ // grant permissions and test
+ assertFalse(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ client.grantSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE);
+ assertTrue(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ client.createTable(stooge, "success", true, TimeType.MILLIS);
+ client.listTables(creds).contains("succcess");
+
+ // revoke permissions
+ client.revokeSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE);
+ assertFalse(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ try {
+ client.createTable(stooge, "fail", true, TimeType.MILLIS);
+ fail("should not create the table");
+ } catch (AccumuloSecurityException ex) {
+ assertFalse(client.listTables(creds).contains("fail"));
+ }
+ // create a table to test table permissions
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ // denied!
+ try {
+ String scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 100);
+ fail("stooge should not read table test");
+ } catch (AccumuloSecurityException ex) {}
+ // grant
+ assertFalse(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ client.grantTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ);
+ assertTrue(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ String scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 10);
+ client.closeScanner(scanner);
+ // revoke
+ client.revokeTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ);
+ assertFalse(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ try {
+ scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 100);
+ fail("stooge should not read table test");
+ } catch (AccumuloSecurityException ex) {}
+
+ // delete user
+ client.dropLocalUser(creds, "stooge");
+ users = client.listLocalUsers(creds);
+ assertEquals(1, users.size());
+
+ }
+
+ @Test
+ public void testBatchWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+
+ WriterOptions writerOptions = new WriterOptions();
+ writerOptions.setLatencyMs(10000);
+ writerOptions.setMaxMemory(2);
+ writerOptions.setThreads(1);
+ writerOptions.setTimeoutMs(100000);
+
+ String batchWriter = client.createWriter(creds, TABLE_TEST, writerOptions);
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ try {
+ client.flush(batchWriter);
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException ex) {}
+ try {
+ client.closeWriter(batchWriter);
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException e) {}
+
+ client.removeConstraint(creds, TABLE_TEST, 2);
+
+ assertScan(new String[][] {}, TABLE_TEST);
+
+ UtilWaitThread.sleep(2000);
+
+ writerOptions = new WriterOptions();
+ writerOptions.setLatencyMs(10000);
+ writerOptions.setMaxMemory(3000);
+ writerOptions.setThreads(1);
+ writerOptions.setTimeoutMs(100000);
+
+ batchWriter = client.createWriter(creds, TABLE_TEST, writerOptions);
+
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ client.flush(batchWriter);
+ client.closeWriter(batchWriter);
+
+ assertScan(new String[][] {{"row1", "cf", "cq", "x"}}, TABLE_TEST);
+
+ client.deleteTable(creds, TABLE_TEST);
+ }
+
+ @Test
+ public void testTableOperations() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ // constraints
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+ assertEquals(2, client.listConstraints(creds, TABLE_TEST).size());
+
+ UtilWaitThread.sleep(2000);
+
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "123"));
+
+ try {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException ex) {}
+
+ client.removeConstraint(creds, TABLE_TEST, 2);
+
+ UtilWaitThread.sleep(2000);
+
+ assertEquals(1, client.listConstraints(creds, TABLE_TEST).size());
+
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
+ assertScan(new String[][] {{"row1", "cf", "cq", "x"}}, TABLE_TEST);
+ // splits, merge
+ client.addSplits(creds, TABLE_TEST, new HashSet<ByteBuffer>(Arrays.asList(s2bb("a"), s2bb("m"), s2bb("z"))));
+ List<ByteBuffer> splits = client.listSplits(creds, TABLE_TEST, 1);
+ assertEquals(Arrays.asList(s2bb("m")), splits);
+ client.mergeTablets(creds, TABLE_TEST, null, s2bb("m"));
+ splits = client.listSplits(creds, TABLE_TEST, 10);
+ assertEquals(Arrays.asList(s2bb("m"), s2bb("z")), splits);
+ client.mergeTablets(creds, TABLE_TEST, null, null);
+ splits = client.listSplits(creds, TABLE_TEST, 10);
+ List<ByteBuffer> empty = Collections.emptyList();
+ assertEquals(empty, splits);
+ // iterators
+ client.deleteTable(creds, TABLE_TEST);
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ HashMap<String,String> options = new HashMap<String,String>();
+ options.put("type", "STRING");
+ options.put("columns", "cf");
+ IteratorSetting setting = new IteratorSetting(10, TABLE_TEST, SummingCombiner.class.getName(), options);
+ client.attachIterator(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
+ for (int i = 0; i < 10; i++) {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "1"));
+ }
+ assertScan(new String[][] {{"row1", "cf", "cq", "10"}}, TABLE_TEST);
+ try {
+ client.checkIteratorConflicts(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
+ fail("checkIteratorConflicts did not throw an exception");
+ } catch (Exception ex) {}
+ client.deleteRows(creds, TABLE_TEST, null, null);
+ client.removeIterator(creds, TABLE_TEST, "test", EnumSet.allOf(IteratorScope.class));
+ String expected[][] = new String[10][];
+ for (int i = 0; i < 10; i++) {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row" + i, "cf", "cq", "" + i));
+ expected[i] = new String[] {"row" + i, "cf", "cq", "" + i};
+ client.flushTable(creds, TABLE_TEST, null, null, true);
+ }
+ assertScan(expected, TABLE_TEST);
+ // clone
+ final String TABLE_TEST2 = makeTableName();
+ client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
+ assertScan(expected, TABLE_TEST2);
+ client.deleteTable(creds, TABLE_TEST2);
+
+ // don't know how to test this, call it just for fun
+ client.clearLocatorCache(creds, TABLE_TEST);
+
+ // compact
+ client.compactTable(creds, TABLE_TEST, null, null, null, true, true);
+ assertEquals(1, countFiles(TABLE_TEST));
+ assertScan(expected, TABLE_TEST);
+
+ // get disk usage
+ client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
+ Set<String> tablesToScan = new HashSet<String>();
+ tablesToScan.add(TABLE_TEST);
+ tablesToScan.add(TABLE_TEST2);
+ tablesToScan.add("foo");
+ client.createTable(creds, "foo", true, TimeType.MILLIS);
+ List<DiskUsage> diskUsage = (client.getDiskUsage(creds, tablesToScan));
+ assertEquals(2, diskUsage.size());
+ assertEquals(1, diskUsage.get(0).getTables().size());
+ assertEquals(2, diskUsage.get(1).getTables().size());
+ client.compactTable(creds, TABLE_TEST2, null, null, null, true, true);
+ diskUsage = (client.getDiskUsage(creds, tablesToScan));
+ assertEquals(3, diskUsage.size());
+ assertEquals(1, diskUsage.get(0).getTables().size());
+ assertEquals(1, diskUsage.get(1).getTables().size());
+ assertEquals(1, diskUsage.get(2).getTables().size());
+ client.deleteTable(creds, "foo");
+ client.deleteTable(creds, TABLE_TEST2);
+
+ // export/import
+ File dir = tempFolder.newFolder("test");
+ File destDir = tempFolder.newFolder("test_dest");
+ client.offlineTable(creds, TABLE_TEST, false);
+ client.exportTable(creds, TABLE_TEST, dir.getAbsolutePath());
+ // copy files to a new location
+ FileSystem fs = FileSystem.get(new Configuration());
+ FSDataInputStream is = fs.open(new Path(dir + "/distcp.txt"));
+ BufferedReader r = new BufferedReader(new InputStreamReader(is));
+ while (true) {
+ String line = r.readLine();
+ if (line == null)
+ break;
+ Path srcPath = new Path(line);
+ FileUtils.copyFile(new File(srcPath.toUri().getPath()), new File(destDir, srcPath.getName()));
+ }
+ client.deleteTable(creds, TABLE_TEST);
+ client.importTable(creds, "testify", destDir.getAbsolutePath());
+ assertScan(expected, "testify");
+ client.deleteTable(creds, "testify");
+
+ try {
+ // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files
+ client.importTable(creds, "testify2", destDir.getAbsolutePath());
+ fail();
+ } catch (Exception e) {}
+
+ assertFalse(client.listTables(creds).contains("testify2"));
+
+ // Locality groups
+ client.createTable(creds, "test", true, TimeType.MILLIS);
+ Map<String,Set<String>> groups = new HashMap<String,Set<String>>();
+ groups.put("group1", Collections.singleton("cf1"));
+ groups.put("group2", Collections.singleton("cf2"));
+ client.setLocalityGroups(creds, "test", groups);
+ assertEquals(groups, client.getLocalityGroups(creds, "test"));
+ // table properties
+ Map<String,String> orig = client.getTableProperties(creds, "test");
+ client.setTableProperty(creds, "test", "table.split.threshold", "500M");
+ Map<String,String> update = client.getTableProperties(creds, "test");
+ assertEquals(update.get("table.split.threshold"), "500M");
+ client.removeTableProperty(creds, "test", "table.split.threshold");
+ update = client.getTableProperties(creds, "test");
+ assertEquals(orig, update);
+ // rename table
+ Map<String,String> tables = client.tableIdMap(creds);
+ client.renameTable(creds, "test", "bar");
+ Map<String,String> tables2 = client.tableIdMap(creds);
+ assertEquals(tables.get("test"), tables2.get("bar"));
+ // table exists
+ assertTrue(client.tableExists(creds, "bar"));
+ assertFalse(client.tableExists(creds, "test"));
+ // bulk import
+ String filename = dir + "/bulk/import/rfile.rf";
+ FileSKVWriter writer = FileOperations.getInstance().openWriter(filename, fs, fs.getConf(), DefaultConfiguration.getInstance());
+ writer.startDefaultLocalityGroup();
+ writer.append(new org.apache.accumulo.core.data.Key(new Text("a"), new Text("b"), new Text("c")), new Value("value".getBytes()));
+ writer.close();
+ fs.mkdirs(new Path(dir + "/bulk/fail"));
+ client.importDirectory(creds, "bar", dir + "/bulk/import", dir + "/bulk/fail", true);
+ String scanner = client.createScanner(creds, "bar", null);
+ ScanResult more = client.nextK(scanner, 100);
+ client.closeScanner(scanner);
+ assertEquals(1, more.results.size());
+ ByteBuffer maxRow = client.getMaxRow(creds, "bar", null, null, false, null, false);
+ assertEquals(s2bb("a"), maxRow);
+
+ assertFalse(client.testTableClassLoad(creds, "bar", "abc123", SortedKeyValueIterator.class.getName()));
+ assertTrue(client.testTableClassLoad(creds, "bar", VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
+ }
+
+ private Condition newCondition(String cf, String cq) {
+ return new Condition(new Column(s2bb(cf), s2bb(cq), s2bb("")));
+ }
+
+ private Condition newCondition(String cf, String cq, String val) {
+ return newCondition(cf, cq).setValue(s2bb(val));
+ }
+
+ private Condition newCondition(String cf, String cq, long ts, String val) {
+ return newCondition(cf, cq).setValue(s2bb(val)).setTimestamp(ts);
+ }
+
+ private ColumnUpdate newColUpdate(String cf, String cq, String val) {
+ return new ColumnUpdate(s2bb(cf), s2bb(cq)).setValue(s2bb(val));
+ }
+
+ private ColumnUpdate newColUpdate(String cf, String cq, long ts, String val) {
+ return new ColumnUpdate(s2bb(cf), s2bb(cq)).setTimestamp(ts).setValue(s2bb(val));
+ }
+
+ private void assertScan(String[][] expected, String table) throws Exception {
+ String scid = client.createScanner(creds, table, new ScanOptions());
+ ScanResult keyValues = client.nextK(scid, expected.length + 1);
+
+ assertEquals(expected.length, keyValues.results.size());
+ assertFalse(keyValues.more);
+
+ for (int i = 0; i < keyValues.results.size(); i++) {
+ checkKey(expected[i][0], expected[i][1], expected[i][2], expected[i][3], keyValues.results.get(i));
+ }
+
+ client.closeScanner(scid);
+ }
+
+ @Test
+ public void testConditionalWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+
+ String cwid = client.createConditionalWriter(creds, TABLE_TEST, new ConditionalWriterOptions());
+
+ Map<ByteBuffer,ConditionalUpdates> updates = new HashMap<ByteBuffer,ConditionalUpdates>();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", 10, "1"),
+ newColUpdate("data", "img", "73435435"))));
+
+ Map<ByteBuffer,ConditionalStatus> results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "73435435"}, {"00345", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test not setting values on conditions
+ updates.clear();
+
+ updates.put(s2bb("00345"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", "2"))));
+ updates.put(s2bb("00346"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", "1"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00345")));
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00346")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "73435435"}, {"00345", "meta", "seq", "1"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting values on conditions
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", "1")), Arrays.asList(newColUpdate("meta", "seq", 20, "2"),
+ newColUpdate("data", "img", "567890"))));
+
+ updates.put(s2bb("00346"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", "2")), Arrays.asList(newColUpdate("meta", "seq", "3"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00346")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "567890"}, {"00345", "meta", "seq", "2"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting timestamp on condition to a non-existant version
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", 10, "2")), Arrays.asList(newColUpdate("meta", "seq", 30, "3"),
+ newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "567890"}, {"00345", "meta", "seq", "2"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting timestamp to an existing version
+
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", 20, "2")), Arrays.asList(newColUpdate("meta", "seq", 30, "3"),
+ newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // run test w/ condition that has iterators
+ // following should fail w/o iterator
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+
+ updates.clear();
+ updates.put(s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "count", "3")), Arrays.asList(newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}}, TABLE_TEST);
+
+ // following test w/ iterator setup should succeed
+ Condition iterCond = newCondition("data", "count", "3");
+ Map<String,String> props = new HashMap<String,String>();
+ props.put("type", "STRING");
+ props.put("columns", "data:count");
+ IteratorSetting is = new IteratorSetting(1, "sumc", SummingCombiner.class.getName(), props);
+ iterCond.setIterators(Arrays.asList(is));
+
+ updates.clear();
+ updates.put(s2bb("00347"), new ConditionalUpdates(Arrays.asList(iterCond), Arrays.asList(newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // test a mutation that violated a constraint
+ updates.clear();
+ updates.put(s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890")), Arrays.asList(newColUpdate("data", "count", "A"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.VIOLATED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // run test with two conditions
+ // both conditions should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "565"), newCondition("data", "count", "2")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // one condition should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890"), newCondition("data", "count", "2")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // one condition should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "565"), newCondition("data", "count", "1")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // both conditions should succeed
+
+ ConditionalStatus result = client.updateRowConditionally(
+ creds,
+ TABLE_TEST,
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890"), newCondition("data", "count", "1")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ assertEquals(ConditionalStatus.ACCEPTED, result);
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}}, TABLE_TEST);
+
+ client.closeConditionalWriter(cwid);
+ try {
+ client.updateRowsConditionally(cwid, updates);
+ fail("conditional writer not closed");
+ } catch (UnknownWriter uk) {}
+
+ // run test with colvis
+ client.createLocalUser(creds, "cwuser", s2bb("bestpasswordever"));
+ client.changeUserAuthorizations(creds, "cwuser", Collections.singleton(s2bb("A")));
+ client.grantTablePermission(creds, "cwuser", TABLE_TEST, TablePermission.WRITE);
+ client.grantTablePermission(creds, "cwuser", TABLE_TEST, TablePermission.READ);
+
+ ByteBuffer cwuCreds = client.login("cwuser", Collections.singletonMap("password", "bestpasswordever"));
+
+ cwid = client.createConditionalWriter(cwuCreds, TABLE_TEST, new ConditionalWriterOptions().setAuthorizations(Collections.singleton(s2bb("A"))));
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A")))), Arrays.asList(newColUpdate("data", "seq", "1"),
+ newColUpdate("data", "c", "1").setColVisibility(s2bb("A")))));
+ updates.put(s2bb("00349"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("B")))), Arrays.asList(newColUpdate("data", "seq", "1"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00348")));
+ assertEquals(ConditionalStatus.INVISIBLE_VISIBILITY, results.get(s2bb("00349")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "1"}}, TABLE_TEST);
+
+ updates.clear();
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A"))).setValue(s2bb("0"))), Arrays.asList(
+ newColUpdate("data", "seq", "2"), newColUpdate("data", "c", "2").setColVisibility(s2bb("A")))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00348")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "1"}}, TABLE_TEST);
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A"))).setValue(s2bb("1"))), Arrays.asList(
+ newColUpdate("data", "seq", "2"), newColUpdate("data", "c", "2").setColVisibility(s2bb("A")))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00348")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "2"}}, TABLE_TEST);
+
+ client.closeConditionalWriter(cwid);
+ try {
+ client.updateRowsConditionally(cwid, updates);
+ fail("conditional writer not closed");
+ } catch (UnknownWriter uk) {}
+
+ client.dropLocalUser(creds, "cwuser");
+
+ }
+
+ private void checkKey(String row, String cf, String cq, String val, KeyValue keyValue) {
+ assertEquals(row, ByteBufferUtil.toString(keyValue.key.row));
+ assertEquals(cf, ByteBufferUtil.toString(keyValue.key.colFamily));
+ assertEquals(cq, ByteBufferUtil.toString(keyValue.key.colQualifier));
+ assertEquals("", ByteBufferUtil.toString(keyValue.key.colVisibility));
+ assertEquals(val, ByteBufferUtil.toString(keyValue.value));
+ }
+
+ // scan metadata for file entries for the given table
+ private int countFiles(String table) throws Exception {
+ Map<String,String> tableIdMap = client.tableIdMap(creds);
+ String tableId = tableIdMap.get(table);
+ Key start = new Key();
+ start.row = s2bb(tableId + ";");
+ Key end = new Key();
+ end.row = s2bb(tableId + "<");
+ end = client.getFollowing(end, PartialKey.ROW);
+ ScanOptions opt = new ScanOptions();
+ opt.range = new Range(start, true, end, false);
+ opt.columns = Collections.singletonList(new ScanColumn(s2bb("file")));
+ String scanner = client.createScanner(creds, MetadataTable.NAME, opt);
+ int result = 0;
+ while (true) {
+ ScanResult more = client.nextK(scanner, 100);
+ result += more.getResults().size();
+ if (!more.more)
+ break;
+ }
+ return result;
+ }
+
+ private Map<ByteBuffer,List<ColumnUpdate>> mutation(String row, String cf, String cq, String value) {
+ ColumnUpdate upd = new ColumnUpdate(s2bb(cf), s2bb(cq));
+ upd.setValue(value.getBytes());
+ return Collections.singletonMap(s2bb(row), Collections.singletonList(upd));
+ }
+
+ private ByteBuffer s2bb(String cf) {
+ return ByteBuffer.wrap(cf.getBytes());
+ }
+
+ private Map<String,String> s2pp(String cf) {
+ Map<String,String> toRet = new TreeMap<String,String>();
+ toRet.put("password", cf);
+ return toRet;
+ }
+
+ static private ByteBuffer t2bb(Text t) {
+ return ByteBuffer.wrap(t.getBytes());
+ }
+
+ @Test
+ public void testGetRowRange() throws Exception {
+ Range range = client.getRowRange(s2bb("xyzzy"));
+ org.apache.accumulo.core.data.Range range2 = new org.apache.accumulo.core.data.Range(new Text("xyzzy"));
+ assertEquals(0, range.start.row.compareTo(t2bb(range2.getStartKey().getRow())));
+ assertEquals(0, range.stop.row.compareTo(t2bb(range2.getEndKey().getRow())));
+ assertEquals(range.startInclusive, range2.isStartKeyInclusive());
+ assertEquals(range.stopInclusive, range2.isEndKeyInclusive());
+ assertEquals(0, range.start.colFamily.compareTo(t2bb(range2.getStartKey().getColumnFamily())));
+ assertEquals(0, range.start.colQualifier.compareTo(t2bb(range2.getStartKey().getColumnQualifier())));
+ assertEquals(0, range.stop.colFamily.compareTo(t2bb(range2.getEndKey().getColumnFamily())));
+ assertEquals(0, range.stop.colQualifier.compareTo(t2bb(range2.getEndKey().getColumnQualifier())));
+ assertEquals(range.start.timestamp, range.start.timestamp);
+ assertEquals(range.stop.timestamp, range.stop.timestamp);
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ }
+}
[12/16] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/da7f937b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/da7f937b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/da7f937b
Branch: refs/heads/master
Commit: da7f937b1d7d88ba7508b22a92ff7d0d110954ae
Parents: 4223d66 4cc25ca
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 18:15:28 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 18:15:28 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +--
.../core/util/shell/ShellConfigTest.java | 5 +-
.../core/util/shell/ShellSetInstanceTest.java | 3 +-
.../accumulo/core/util/shell/ShellTest.java | 5 +-
.../apache/accumulo/proxy/SimpleProxyIT.java | 98 +++++++++----
.../org/apache/accumulo/test/ShellServerIT.java | 145 ++++++++++++++-----
6 files changed, 196 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 420d465,6943b37..0c2cf3c
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -219,111 -200,114 +219,109 @@@ public class Shell extends ShellOption
private boolean logErrorsToConsole = false;
private PrintWriter writer = null;
private boolean masking = false;
-
+
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
-
+
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
-
+
// Not for client use
public boolean config(String... args) {
-
- CommandLine cl;
+ ShellOptionsJC options = new ShellOptionsJC();
+ JCommander jc = new JCommander();
+
+ jc.setProgramName("accumulo shell");
+ jc.addObject(options);
try {
- cl = new BasicParser().parse(opts, args);
- if (cl.getArgs().length > 0)
- throw new ParseException("Unrecognized arguments: " + cl.getArgList());
-
- if (cl.hasOption(helpOpt.getOpt())) {
- configError = true;
- printHelp("shell", SHELL_DESCRIPTION, opts);
- return true;
- }
-
- setDebugging(cl.hasOption(debugOption.getLongOpt()));
- authTimeout = Integer.parseInt(cl.getOptionValue(authTimeoutOpt.getLongOpt(), DEFAULT_AUTH_TIMEOUT)) * 60 * 1000;
- disableAuthTimeout = cl.hasOption(disableAuthTimeoutOpt.getLongOpt());
-
- if (cl.hasOption(zooKeeperInstance.getOpt()) && cl.getOptionValues(zooKeeperInstance.getOpt()).length != 2)
- throw new MissingArgumentException(zooKeeperInstance);
-
- } catch (Exception e) {
+ jc.parse(args);
+ } catch (ParameterException e) {
configError = true;
- printException(e);
- printHelp("shell", SHELL_DESCRIPTION, opts);
+ }
+
+ if (options.isHelpEnabled()) {
+ configError = true;
+ }
+
+ if (!configError && options.getUnrecognizedOptions() != null) {
+ configError = true;
+ logError("Unrecognized Options: " + options.getUnrecognizedOptions().toString());
+ }
+
+ if (configError) {
+ jc.usage();
return true;
}
-
+
+ setDebugging(options.isDebugEnabled());
+ authTimeout = options.getAuthTimeout() * 60 * 1000; // convert minutes to milliseconds
+ disableAuthTimeout = options.isAuthTimeoutDisabled();
+
// get the options that were parsed
- String sysUser = System.getProperty("user.name");
- if (sysUser == null)
- sysUser = "root";
- String user = cl.getOptionValue(usernameOption.getOpt(), sysUser);
-
- String passw = cl.getOptionValue(passwOption.getOpt(), null);
- tabCompletion = !cl.hasOption(tabCompleteOption.getLongOpt());
- String[] loginOptions = cl.getOptionValues(loginOption.getOpt());
-
+ String user = options.getUsername();
+ String password = options.getPassword();
+
+ tabCompletion = !options.isTabCompletionDisabled();
+
// Use a fake (Mock), ZK, or HdfsZK Accumulo instance
- setInstance(cl);
-
+ setInstance(options);
+
+ // AuthenticationToken options
+ token = options.getAuthenticationToken();
+ Map<String,String> loginOptions = options.getTokenProperties();
+
// process default parameters if unspecified
try {
- if (loginOptions != null && !cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Must supply '-" + tokenOption.getOpt() + "' option with '-" + loginOption.getOpt() + "' option");
-
- if (loginOptions == null && cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Must supply '-" + loginOption.getOpt() + "' option with '-" + tokenOption.getOpt() + "' option");
-
- if (passw != null && cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Can not supply '-" + passwOption.getOpt() + "' option with '-" + tokenOption.getOpt() + "' option");
-
- if (user == null)
- throw new MissingArgumentException(usernameOption);
-
- if (loginOptions != null && cl.hasOption(tokenOption.getOpt())) {
- Properties props = new Properties();
- for (String loginOption : loginOptions)
- for (String lo : loginOption.split(",")) {
- String[] split = lo.split("=");
- props.put(split[0], split[1]);
- }
-
- this.token = Class.forName(cl.getOptionValue(tokenOption.getOpt())).asSubclass(AuthenticationToken.class).newInstance();
- this.token.init(props);
- }
-
- if (!cl.hasOption(fakeOption.getLongOpt())) {
- DistributedTrace.enable(instance, new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()), "shell", InetAddress.getLocalHost()
- .getHostName());
+ boolean hasToken = (token != null);
+ boolean hasTokenOptions = !loginOptions.isEmpty();
+
+ if (hasToken && password != null) {
+ throw new ParameterException("Can not supply '--pass' option with '--tokenClass' option");
}
-
+
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
- public void start() {
- reader.getTerminal().enableEcho();
+ public void run() {
+ reader.getTerminal().setEchoEnabled(true);
}
});
-
- if (passw != null) {
- this.token = new PasswordToken(passw);
+
+ // Need either both a token and options, or neither, but not just one.
+ if (hasToken != hasTokenOptions) {
+ throw new ParameterException("Must supply either both or neither of '--tokenClass' and '--tokenProperty'");
+ } else if (hasToken) { // implied hasTokenOptions
+ // Fully qualified name so we don't shadow java.util.Properties
+ org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties props;
+ // and line wrap it because the package name is so long
+ props = new org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties();
+
+ props.putAllStrings(loginOptions);
+ token.init(props);
+ } else {
+ // Read password if the user explicitly asked for it, or didn't specify anything at all
+ if ("stdin".equals(password) || password == null) {
+ password = reader.readLine("Password: ", '*');
+ }
+
+ if (password == null) {
+ // User cancel, e.g. Ctrl-D pressed
+ throw new ParameterException("No password or token option supplied");
+ } else {
+ this.token = new PasswordToken(password);
+ }
}
-
- if (this.token == null) {
- passw = readMaskedLine("Password: ", '*');
- if (passw != null)
- this.token = new PasswordToken(passw);
+
+ if (!options.isFake()) {
+ ZooReader zr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ DistributedTrace.enable(instance, zr, "shell", InetAddress.getLocalHost().getHostName());
}
-
- if (this.token == null) {
- reader.printNewline();
- throw new MissingArgumentException("No password or token option supplied");
- } // user canceled
-
+
this.setTableName("");
this.principal = user;
connector = instance.getConnector(this.principal, token);
@@@ -1053,17 -939,13 +1051,13 @@@
private final void printHelp(String usage, String description, Options opts) {
printHelp(usage, description, opts, Integer.MAX_VALUE);
}
-
+
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
-
+
public int getExitCode() {
return exitCode;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
index c9914da,0000000..43d2e12
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
@@@ -1,89 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.PrintStream;
++import java.io.PrintWriter;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.shell.ShellTest.TestOutputStream;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.beust.jcommander.ParameterException;
+
+public class ShellConfigTest {
+ TestOutputStream output;
+ Shell shell;
+ PrintStream out;
+
+ @Before
+ public void setUp() throws Exception {
+ Shell.log.setLevel(Level.ERROR);
+
+ out = System.out;
+ output = new TestOutputStream();
+ System.setOut(new PrintStream(output));
-
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output));
++
++ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output), new PrintWriter(output));
+ shell.setLogErrorsToConsole();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ shell.shutdown();
+ output.clear();
+ System.setOut(out);
+ }
+
+ @Test
+ public void testHelp() {
+ assertTrue(shell.config("--help"));
+ assertTrue("Did not print usage", output.get().startsWith("Usage"));
+ }
+
+ @Test
+ public void testBadArg() {
+ assertTrue(shell.config("--bogus"));
+ assertTrue("Did not print usage", output.get().startsWith("Usage"));
+ }
+
+ @Test
+ public void testToken() {
+ assertTrue(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName()));
+ assertTrue(output.get().contains(ParameterException.class.getCanonicalName()));
+ }
+
+ @Test
+ public void testTokenAndOption() {
+ assertFalse(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName(), "-u", "foo", "-l", "password=foo"));
+ }
+
+ @Test
+ public void testTokenAndOptionAndPassword() {
+ assertTrue(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName(), "-l", "password=foo", "-p", "bar"));
+ assertTrue(output.get().contains(ParameterException.class.getCanonicalName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
index 2929b04,0000000..ad24e6d
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
@@@ -1,240 -1,0 +1,241 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.powermock.api.easymock.PowerMock.createMock;
+import static org.powermock.api.easymock.PowerMock.expectLastCall;
+import static org.powermock.api.easymock.PowerMock.expectNew;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verify;
+
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
++import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigSanityCheck;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class})
+public class ShellSetInstanceTest {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass() {
+ // This is necessary because PowerMock messes with Hadoop's ability to
+ // determine the current user (see security.UserGroupInformation).
+ System.setProperty("HADOOP_USER_NAME", "test");
+ }
+ @AfterClass
+ public static void teardownClass() {
+ System.clearProperty("HADOOP_USER_NAME");
+ }
+
+ private TestOutputStream output;
+ private Shell shell;
+
+ @Before
+ public void setup() throws IOException {
+ Shell.log.setLevel(Level.OFF);
+ output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output));
++ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output), new PrintWriter(output));
+ shell.setLogErrorsToConsole();
+ }
+ @After
+ public void tearDown() {
+ shell.shutdown();
+ SiteConfiguration.clearInstance();
+ }
+
+ @Test
+ public void testSetInstance_Fake() throws Exception {
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(true);
+ replay(opts);
+ MockInstance theInstance = createMock(MockInstance.class);
+ expectNew(MockInstance.class, "fake").andReturn(theInstance);
+ replay(theInstance, MockInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, MockInstance.class);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_Explicit() throws Exception {
+ testSetInstance_HdfsZooInstance(true, false, false);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_InstanceGiven() throws Exception {
+ testSetInstance_HdfsZooInstance(false, true, false);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_HostsGiven() throws Exception {
+ testSetInstance_HdfsZooInstance(false, false, true);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_Implicit() throws Exception {
+ testSetInstance_HdfsZooInstance(false, false, false);
+ }
+
+ private void testSetInstance_HdfsZooInstance(boolean explicitHdfs, boolean onlyInstance, boolean onlyHosts)
+ throws Exception {
+ ClientConfiguration clientConf = createMock(ClientConfiguration.class);
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(false);
+ expect(opts.getClientConfiguration()).andReturn(clientConf);
+ expect(opts.isHdfsZooInstance()).andReturn(explicitHdfs);
+ if (!explicitHdfs) {
+ expect(opts.getZooKeeperInstance())
+ .andReturn(Collections.<String>emptyList());
+ if (onlyInstance) {
+ expect(opts.getZooKeeperInstanceName()).andReturn("instance");
+ expect(clientConf.withInstance("instance")).andReturn(clientConf);
+ } else {
+ expect(opts.getZooKeeperInstanceName()).andReturn(null);
+ }
+ if (onlyHosts) {
+ expect(opts.getZooKeeperHosts()).andReturn("host3,host4");
+ expect(clientConf.withZkHosts("host3,host4")).andReturn(clientConf);
+ } else {
+ expect(opts.getZooKeeperHosts()).andReturn(null);
+ }
+ }
+ replay(opts);
+
+ if (!onlyInstance) {
+ expect(clientConf.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
+ }
+
+ mockStatic(ConfigSanityCheck.class);
+ ConfigSanityCheck.validate(EasyMock.<AccumuloConfiguration>anyObject());
+ expectLastCall().atLeastOnce();
+ replay(ConfigSanityCheck.class);
+
+ if (!onlyHosts) {
+ expect(clientConf.containsKey(Property.INSTANCE_ZK_HOST.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_ZK_HOST.getKey())).andReturn("host1,host2").atLeastOnce();
+ expect(clientConf.withZkHosts("host1,host2")).andReturn(clientConf);
+ }
+ if (!onlyInstance) {
+ expect(clientConf.containsKey(Property.INSTANCE_VOLUMES.getKey())).andReturn(false).atLeastOnce();
+ expect(clientConf.containsKey(Property.INSTANCE_DFS_DIR.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.containsKey(Property.INSTANCE_DFS_URI.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_DFS_URI.getKey())).andReturn("hdfs://nn1").atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_DFS_DIR.getKey())).andReturn("/dfs").atLeastOnce();
+ }
+
+ UUID randomUUID = null;
+ if (!onlyInstance) {
+ mockStatic(ZooUtil.class);
+ randomUUID = UUID.randomUUID();
+ expect(ZooUtil.getInstanceIDFromHdfs(anyObject(Path.class), anyObject(AccumuloConfiguration.class)))
+ .andReturn(randomUUID.toString());
+ replay(ZooUtil.class);
+ expect(clientConf.withInstance(randomUUID)).andReturn(clientConf);
+ }
+ replay(clientConf);
+
+ ZooKeeperInstance theInstance = createMock(ZooKeeperInstance.class);
+
+ expectNew(ZooKeeperInstance.class, clientConf).andReturn(theInstance);
+ replay(theInstance, ZooKeeperInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, ZooKeeperInstance.class);
+ }
+ @Test
+ public void testSetInstance_ZKInstance_DashZ() throws Exception {
+ testSetInstance_ZKInstance(true);
+ }
+ @Test
+ public void testSetInstance_ZKInstance_DashZIandZH() throws Exception {
+ testSetInstance_ZKInstance(false);
+ }
+ private void testSetInstance_ZKInstance(boolean dashZ) throws Exception {
+ ClientConfiguration clientConf = createMock(ClientConfiguration.class);
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(false);
+ expect(opts.getClientConfiguration()).andReturn(clientConf);
+ expect(opts.isHdfsZooInstance()).andReturn(false);
+ if (dashZ) {
+ expect(clientConf.withInstance("foo")).andReturn(clientConf);
+ expect(clientConf.withZkHosts("host1,host2")).andReturn(clientConf);
+ List<String> zl = new java.util.ArrayList<String>();
+ zl.add("foo");
+ zl.add("host1,host2");
+ expect(opts.getZooKeeperInstance()).andReturn(zl);
+ expectLastCall().anyTimes();
+ } else {
+ expect(clientConf.withInstance("bar")).andReturn(clientConf);
+ expect(clientConf.withZkHosts("host3,host4")).andReturn(clientConf);
+ expect(opts.getZooKeeperInstance()).andReturn(Collections.<String>emptyList());
+ expect(opts.getZooKeeperInstanceName()).andReturn("bar");
+ expect(opts.getZooKeeperHosts()).andReturn("host3,host4");
+ }
+ replay(clientConf);
+ replay(opts);
+
+ ZooKeeperInstance theInstance = createMock(ZooKeeperInstance.class);
+ expectNew(ZooKeeperInstance.class, clientConf).andReturn(theInstance);
+ replay(theInstance, ZooKeeperInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, ZooKeeperInstance.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 4771f2c,8505370..82d1f34
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@@ -19,18 -19,16 +19,20 @@@ package org.apache.accumulo.core.util.s
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
+ import java.io.OutputStreamWriter;
+ import java.io.PrintWriter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
+import org.apache.accumulo.core.util.format.DateStringFormatter;
import org.apache.log4j.Level;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@@ -103,8 -82,8 +105,9 @@@ public class ShellTest
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
+ input = new StringInputStream();
- shell = new Shell(new ConsoleReader(input, output));
++ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
++ shell = new Shell(new ConsoleReader(input, output), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
[05/16] git commit: ACCUMULO-1936 Make testInstanceOperations a more
reliable test.
Posted by el...@apache.org.
ACCUMULO-1936 Make testInstanceOperations a more reliable test.
Avoid pulling scans for users other than root and compactions for
compactions over tables other than the one we're looking at. Slow down
the slow iterator even more.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/72395e01
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/72395e01
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/72395e01
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 72395e010d3f495507d03664057a63c3d690a206
Parents: e13c273
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:26:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/proxy/SimpleTest.java | 110 +++++++++++++------
1 file changed, 75 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/72395e01/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 5573d20..77e7b9e 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServer;
@@ -102,7 +103,8 @@ import org.junit.rules.TemporaryFolder;
* Call every method on the proxy and try to verify that it works.
*/
public class SimpleTest {
-
+ private static final Logger log = Logger.getLogger(SimpleTest.class);
+
public static TemporaryFolder folder = new TemporaryFolder();
public static final String TABLE_TEST = "test";
@@ -704,7 +706,7 @@ public class SimpleTest {
assertEquals(0, entries.results.size());
}
- @Test(timeout = 10000)
+ @Test(timeout = 60000)
public void testInstanceOperations() throws Exception {
int tservers = 0;
for (String tserver : client.getTabletServers(creds)) {
@@ -745,12 +747,14 @@ public class SimpleTest {
// create a table that's very slow, so we can look for scans/compactions
client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ // Should take 5 seconds to read every record
+ for (int i = 0; i < 20; i++) {
+ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
+ }
+
+ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
// scan
Thread t = new Thread() {
@@ -768,28 +772,45 @@ public class SimpleTest {
}
};
t.start();
+
// look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
+ List<ActiveScan> scans = new ArrayList<ActiveScan>();
+ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
+ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
+ for (ActiveScan scan : scansForServer) {
+ if ("root".equals(scan.getUser())) {
+ scans.add(scan);
+ }
+ }
+
if (!scans.isEmpty())
- break loop;
+ break;
UtilWaitThread.sleep(10);
}
}
t.join();
- assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
+ assertFalse(scans.isEmpty());
+ boolean found = false;
+ Map<String,String> map = null;
+ for (int i = 0; i < scans.size() && !found; i++) {
+ ActiveScan scan = scans.get(i);
+ if ("root".equals(scan.getUser())) {
+ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
+ assertEquals(ScanType.SINGLE, scan.getType());
+ assertEquals("slow", scan.getTable());
+
+ map = client.tableIdMap(creds);
+ assertEquals(map.get("slow"), scan.getExtent().tableId);
+ assertTrue(scan.getExtent().endRow == null);
+ assertTrue(scan.getExtent().prevEndRow == null);
+ found = true;
+ }
+ }
+
+ assertTrue("Could not find a scan against the 'slow' table", found);
+
// start a compaction
t = new Thread() {
@Override
@@ -803,27 +824,46 @@ public class SimpleTest {
}
};
t.start();
-
+
+ final String desiredTableId = map.get("slow");
+
// try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
+ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
+ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
+ // Iterate over the tservers
for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
+ // And get the compactions on each
+ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
+ for (ActiveCompaction compact : compactionsOnServer) {
+ // There might be other compactions occurring (e.g. on METADATA) in which
+ // case we want to prune out those that aren't for our slow table
+ if (desiredTableId.equals(compact.getExtent().tableId)) {
+ compactions.add(compact);
+ }
+ }
+
+ // If we found a compaction for the table we wanted, so we can stop looking
if (!compactions.isEmpty())
- break loop2;
+ break;
}
UtilWaitThread.sleep(10);
}
t.join();
+
// verify the compaction information
assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
+ for (ActiveCompaction c : compactions) {
+ if (desiredTableId.equals(c.getExtent().tableId)) {
+ assertTrue(c.inputFiles.isEmpty());
+ assertEquals(CompactionType.MINOR, c.getType());
+ assertEquals(CompactionReason.USER, c.getReason());
+ assertEquals("", c.localityGroup);
+ assertTrue(c.outputFile.contains("default_tablet"));
+
+ return;
+ }
+ }
+ fail("Expection to find running compaction for table 'slow' but did not find one");
}
@Test
@@ -1178,5 +1218,5 @@ public class SimpleTest {
accumulo.stop();
folder.delete();
}
-
-}
+
+}
\ No newline at end of file
[15/16] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/da7f937b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/da7f937b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/da7f937b
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: da7f937b1d7d88ba7508b22a92ff7d0d110954ae
Parents: 4223d66 4cc25ca
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 18:15:28 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 18:15:28 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +--
.../core/util/shell/ShellConfigTest.java | 5 +-
.../core/util/shell/ShellSetInstanceTest.java | 3 +-
.../accumulo/core/util/shell/ShellTest.java | 5 +-
.../apache/accumulo/proxy/SimpleProxyIT.java | 98 +++++++++----
.../org/apache/accumulo/test/ShellServerIT.java | 145 ++++++++++++++-----
6 files changed, 196 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 420d465,6943b37..0c2cf3c
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -219,111 -200,114 +219,109 @@@ public class Shell extends ShellOption
private boolean logErrorsToConsole = false;
private PrintWriter writer = null;
private boolean masking = false;
-
+
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
-
+
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
-
+
// Not for client use
public boolean config(String... args) {
-
- CommandLine cl;
+ ShellOptionsJC options = new ShellOptionsJC();
+ JCommander jc = new JCommander();
+
+ jc.setProgramName("accumulo shell");
+ jc.addObject(options);
try {
- cl = new BasicParser().parse(opts, args);
- if (cl.getArgs().length > 0)
- throw new ParseException("Unrecognized arguments: " + cl.getArgList());
-
- if (cl.hasOption(helpOpt.getOpt())) {
- configError = true;
- printHelp("shell", SHELL_DESCRIPTION, opts);
- return true;
- }
-
- setDebugging(cl.hasOption(debugOption.getLongOpt()));
- authTimeout = Integer.parseInt(cl.getOptionValue(authTimeoutOpt.getLongOpt(), DEFAULT_AUTH_TIMEOUT)) * 60 * 1000;
- disableAuthTimeout = cl.hasOption(disableAuthTimeoutOpt.getLongOpt());
-
- if (cl.hasOption(zooKeeperInstance.getOpt()) && cl.getOptionValues(zooKeeperInstance.getOpt()).length != 2)
- throw new MissingArgumentException(zooKeeperInstance);
-
- } catch (Exception e) {
+ jc.parse(args);
+ } catch (ParameterException e) {
configError = true;
- printException(e);
- printHelp("shell", SHELL_DESCRIPTION, opts);
+ }
+
+ if (options.isHelpEnabled()) {
+ configError = true;
+ }
+
+ if (!configError && options.getUnrecognizedOptions() != null) {
+ configError = true;
+ logError("Unrecognized Options: " + options.getUnrecognizedOptions().toString());
+ }
+
+ if (configError) {
+ jc.usage();
return true;
}
-
+
+ setDebugging(options.isDebugEnabled());
+ authTimeout = options.getAuthTimeout() * 60 * 1000; // convert minutes to milliseconds
+ disableAuthTimeout = options.isAuthTimeoutDisabled();
+
// get the options that were parsed
- String sysUser = System.getProperty("user.name");
- if (sysUser == null)
- sysUser = "root";
- String user = cl.getOptionValue(usernameOption.getOpt(), sysUser);
-
- String passw = cl.getOptionValue(passwOption.getOpt(), null);
- tabCompletion = !cl.hasOption(tabCompleteOption.getLongOpt());
- String[] loginOptions = cl.getOptionValues(loginOption.getOpt());
-
+ String user = options.getUsername();
+ String password = options.getPassword();
+
+ tabCompletion = !options.isTabCompletionDisabled();
+
// Use a fake (Mock), ZK, or HdfsZK Accumulo instance
- setInstance(cl);
-
+ setInstance(options);
+
+ // AuthenticationToken options
+ token = options.getAuthenticationToken();
+ Map<String,String> loginOptions = options.getTokenProperties();
+
// process default parameters if unspecified
try {
- if (loginOptions != null && !cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Must supply '-" + tokenOption.getOpt() + "' option with '-" + loginOption.getOpt() + "' option");
-
- if (loginOptions == null && cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Must supply '-" + loginOption.getOpt() + "' option with '-" + tokenOption.getOpt() + "' option");
-
- if (passw != null && cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Can not supply '-" + passwOption.getOpt() + "' option with '-" + tokenOption.getOpt() + "' option");
-
- if (user == null)
- throw new MissingArgumentException(usernameOption);
-
- if (loginOptions != null && cl.hasOption(tokenOption.getOpt())) {
- Properties props = new Properties();
- for (String loginOption : loginOptions)
- for (String lo : loginOption.split(",")) {
- String[] split = lo.split("=");
- props.put(split[0], split[1]);
- }
-
- this.token = Class.forName(cl.getOptionValue(tokenOption.getOpt())).asSubclass(AuthenticationToken.class).newInstance();
- this.token.init(props);
- }
-
- if (!cl.hasOption(fakeOption.getLongOpt())) {
- DistributedTrace.enable(instance, new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()), "shell", InetAddress.getLocalHost()
- .getHostName());
+ boolean hasToken = (token != null);
+ boolean hasTokenOptions = !loginOptions.isEmpty();
+
+ if (hasToken && password != null) {
+ throw new ParameterException("Can not supply '--pass' option with '--tokenClass' option");
}
-
+
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
- public void start() {
- reader.getTerminal().enableEcho();
+ public void run() {
+ reader.getTerminal().setEchoEnabled(true);
}
});
-
- if (passw != null) {
- this.token = new PasswordToken(passw);
+
+ // Need either both a token and options, or neither, but not just one.
+ if (hasToken != hasTokenOptions) {
+ throw new ParameterException("Must supply either both or neither of '--tokenClass' and '--tokenProperty'");
+ } else if (hasToken) { // implied hasTokenOptions
+ // Fully qualified name so we don't shadow java.util.Properties
+ org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties props;
+ // and line wrap it because the package name is so long
+ props = new org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties();
+
+ props.putAllStrings(loginOptions);
+ token.init(props);
+ } else {
+ // Read password if the user explicitly asked for it, or didn't specify anything at all
+ if ("stdin".equals(password) || password == null) {
+ password = reader.readLine("Password: ", '*');
+ }
+
+ if (password == null) {
+ // User cancel, e.g. Ctrl-D pressed
+ throw new ParameterException("No password or token option supplied");
+ } else {
+ this.token = new PasswordToken(password);
+ }
}
-
- if (this.token == null) {
- passw = readMaskedLine("Password: ", '*');
- if (passw != null)
- this.token = new PasswordToken(passw);
+
+ if (!options.isFake()) {
+ ZooReader zr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ DistributedTrace.enable(instance, zr, "shell", InetAddress.getLocalHost().getHostName());
}
-
- if (this.token == null) {
- reader.printNewline();
- throw new MissingArgumentException("No password or token option supplied");
- } // user canceled
-
+
this.setTableName("");
this.principal = user;
connector = instance.getConnector(this.principal, token);
@@@ -1053,17 -939,13 +1051,13 @@@
private final void printHelp(String usage, String description, Options opts) {
printHelp(usage, description, opts, Integer.MAX_VALUE);
}
-
+
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
-
+
public int getExitCode() {
return exitCode;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
index c9914da,0000000..43d2e12
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
@@@ -1,89 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.PrintStream;
++import java.io.PrintWriter;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.shell.ShellTest.TestOutputStream;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.beust.jcommander.ParameterException;
+
+public class ShellConfigTest {
+ TestOutputStream output;
+ Shell shell;
+ PrintStream out;
+
+ @Before
+ public void setUp() throws Exception {
+ Shell.log.setLevel(Level.ERROR);
+
+ out = System.out;
+ output = new TestOutputStream();
+ System.setOut(new PrintStream(output));
-
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output));
++
++ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output), new PrintWriter(output));
+ shell.setLogErrorsToConsole();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ shell.shutdown();
+ output.clear();
+ System.setOut(out);
+ }
+
+ @Test
+ public void testHelp() {
+ assertTrue(shell.config("--help"));
+ assertTrue("Did not print usage", output.get().startsWith("Usage"));
+ }
+
+ @Test
+ public void testBadArg() {
+ assertTrue(shell.config("--bogus"));
+ assertTrue("Did not print usage", output.get().startsWith("Usage"));
+ }
+
+ @Test
+ public void testToken() {
+ assertTrue(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName()));
+ assertTrue(output.get().contains(ParameterException.class.getCanonicalName()));
+ }
+
+ @Test
+ public void testTokenAndOption() {
+ assertFalse(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName(), "-u", "foo", "-l", "password=foo"));
+ }
+
+ @Test
+ public void testTokenAndOptionAndPassword() {
+ assertTrue(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName(), "-l", "password=foo", "-p", "bar"));
+ assertTrue(output.get().contains(ParameterException.class.getCanonicalName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
index 2929b04,0000000..ad24e6d
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
@@@ -1,240 -1,0 +1,241 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.powermock.api.easymock.PowerMock.createMock;
+import static org.powermock.api.easymock.PowerMock.expectLastCall;
+import static org.powermock.api.easymock.PowerMock.expectNew;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verify;
+
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
++import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigSanityCheck;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class})
+public class ShellSetInstanceTest {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass() {
+ // This is necessary because PowerMock messes with Hadoop's ability to
+ // determine the current user (see security.UserGroupInformation).
+ System.setProperty("HADOOP_USER_NAME", "test");
+ }
+ @AfterClass
+ public static void teardownClass() {
+ System.clearProperty("HADOOP_USER_NAME");
+ }
+
+ private TestOutputStream output;
+ private Shell shell;
+
+ @Before
+ public void setup() throws IOException {
+ Shell.log.setLevel(Level.OFF);
+ output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output));
++ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output), new PrintWriter(output));
+ shell.setLogErrorsToConsole();
+ }
+ @After
+ public void tearDown() {
+ shell.shutdown();
+ SiteConfiguration.clearInstance();
+ }
+
+ @Test
+ public void testSetInstance_Fake() throws Exception {
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(true);
+ replay(opts);
+ MockInstance theInstance = createMock(MockInstance.class);
+ expectNew(MockInstance.class, "fake").andReturn(theInstance);
+ replay(theInstance, MockInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, MockInstance.class);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_Explicit() throws Exception {
+ testSetInstance_HdfsZooInstance(true, false, false);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_InstanceGiven() throws Exception {
+ testSetInstance_HdfsZooInstance(false, true, false);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_HostsGiven() throws Exception {
+ testSetInstance_HdfsZooInstance(false, false, true);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_Implicit() throws Exception {
+ testSetInstance_HdfsZooInstance(false, false, false);
+ }
+
+ private void testSetInstance_HdfsZooInstance(boolean explicitHdfs, boolean onlyInstance, boolean onlyHosts)
+ throws Exception {
+ ClientConfiguration clientConf = createMock(ClientConfiguration.class);
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(false);
+ expect(opts.getClientConfiguration()).andReturn(clientConf);
+ expect(opts.isHdfsZooInstance()).andReturn(explicitHdfs);
+ if (!explicitHdfs) {
+ expect(opts.getZooKeeperInstance())
+ .andReturn(Collections.<String>emptyList());
+ if (onlyInstance) {
+ expect(opts.getZooKeeperInstanceName()).andReturn("instance");
+ expect(clientConf.withInstance("instance")).andReturn(clientConf);
+ } else {
+ expect(opts.getZooKeeperInstanceName()).andReturn(null);
+ }
+ if (onlyHosts) {
+ expect(opts.getZooKeeperHosts()).andReturn("host3,host4");
+ expect(clientConf.withZkHosts("host3,host4")).andReturn(clientConf);
+ } else {
+ expect(opts.getZooKeeperHosts()).andReturn(null);
+ }
+ }
+ replay(opts);
+
+ if (!onlyInstance) {
+ expect(clientConf.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
+ }
+
+ mockStatic(ConfigSanityCheck.class);
+ ConfigSanityCheck.validate(EasyMock.<AccumuloConfiguration>anyObject());
+ expectLastCall().atLeastOnce();
+ replay(ConfigSanityCheck.class);
+
+ if (!onlyHosts) {
+ expect(clientConf.containsKey(Property.INSTANCE_ZK_HOST.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_ZK_HOST.getKey())).andReturn("host1,host2").atLeastOnce();
+ expect(clientConf.withZkHosts("host1,host2")).andReturn(clientConf);
+ }
+ if (!onlyInstance) {
+ expect(clientConf.containsKey(Property.INSTANCE_VOLUMES.getKey())).andReturn(false).atLeastOnce();
+ expect(clientConf.containsKey(Property.INSTANCE_DFS_DIR.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.containsKey(Property.INSTANCE_DFS_URI.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_DFS_URI.getKey())).andReturn("hdfs://nn1").atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_DFS_DIR.getKey())).andReturn("/dfs").atLeastOnce();
+ }
+
+ UUID randomUUID = null;
+ if (!onlyInstance) {
+ mockStatic(ZooUtil.class);
+ randomUUID = UUID.randomUUID();
+ expect(ZooUtil.getInstanceIDFromHdfs(anyObject(Path.class), anyObject(AccumuloConfiguration.class)))
+ .andReturn(randomUUID.toString());
+ replay(ZooUtil.class);
+ expect(clientConf.withInstance(randomUUID)).andReturn(clientConf);
+ }
+ replay(clientConf);
+
+ ZooKeeperInstance theInstance = createMock(ZooKeeperInstance.class);
+
+ expectNew(ZooKeeperInstance.class, clientConf).andReturn(theInstance);
+ replay(theInstance, ZooKeeperInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, ZooKeeperInstance.class);
+ }
+ @Test
+ public void testSetInstance_ZKInstance_DashZ() throws Exception {
+ testSetInstance_ZKInstance(true);
+ }
+ @Test
+ public void testSetInstance_ZKInstance_DashZIandZH() throws Exception {
+ testSetInstance_ZKInstance(false);
+ }
+ private void testSetInstance_ZKInstance(boolean dashZ) throws Exception {
+ ClientConfiguration clientConf = createMock(ClientConfiguration.class);
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(false);
+ expect(opts.getClientConfiguration()).andReturn(clientConf);
+ expect(opts.isHdfsZooInstance()).andReturn(false);
+ if (dashZ) {
+ expect(clientConf.withInstance("foo")).andReturn(clientConf);
+ expect(clientConf.withZkHosts("host1,host2")).andReturn(clientConf);
+ List<String> zl = new java.util.ArrayList<String>();
+ zl.add("foo");
+ zl.add("host1,host2");
+ expect(opts.getZooKeeperInstance()).andReturn(zl);
+ expectLastCall().anyTimes();
+ } else {
+ expect(clientConf.withInstance("bar")).andReturn(clientConf);
+ expect(clientConf.withZkHosts("host3,host4")).andReturn(clientConf);
+ expect(opts.getZooKeeperInstance()).andReturn(Collections.<String>emptyList());
+ expect(opts.getZooKeeperInstanceName()).andReturn("bar");
+ expect(opts.getZooKeeperHosts()).andReturn("host3,host4");
+ }
+ replay(clientConf);
+ replay(opts);
+
+ ZooKeeperInstance theInstance = createMock(ZooKeeperInstance.class);
+ expectNew(ZooKeeperInstance.class, clientConf).andReturn(theInstance);
+ replay(theInstance, ZooKeeperInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, ZooKeeperInstance.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 4771f2c,8505370..82d1f34
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@@ -19,18 -19,16 +19,20 @@@ package org.apache.accumulo.core.util.s
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
+ import java.io.OutputStreamWriter;
+ import java.io.PrintWriter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
+import org.apache.accumulo.core.util.format.DateStringFormatter;
import org.apache.log4j.Level;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@@ -103,8 -82,8 +105,9 @@@ public class ShellTest
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
+ input = new StringInputStream();
- shell = new Shell(new ConsoleReader(input, output));
++ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
++ shell = new Shell(new ConsoleReader(input, output), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
[08/16] git commit: ACCUMULO-2433 Add a way to still use the exec
methods but provide a custom error message on failure.
Posted by el...@apache.org.
ACCUMULO-2433 Add a way to still use the exec methods but provide a custom error message on failure.
Fix some tests that I've seen fail to provide better messages on failure to help future debugging
because I can't get them reproduced locally in any reasonable timeframe.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4cc25caa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4cc25caa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4cc25caa
Branch: refs/heads/master
Commit: 4cc25caa07ea5ca49e4d81f9e5a22908e45bba59
Parents: 72395e0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:59:09 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 17:13:22 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/test/ShellServerTest.java | 128 ++++++++++++++-----
1 file changed, 99 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4cc25caa/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index e2ebbcf..d66f509 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -31,6 +31,10 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -53,6 +57,7 @@ import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.server.trace.TraceServer;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -85,7 +90,19 @@ public class ShellServerTest {
}
}
+ private static abstract class ErrorMessageCallback {
+ public abstract String getErrorMessage();
+ }
+
+ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
+ private static final String empty = "";
+ public String getErrorMessage() {
+ return empty;
+ }
+ }
+
private static final Logger log = Logger.getLogger(ShellServerTest.class);
+ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
@@ -104,36 +121,51 @@ public class ShellServerTest {
}
String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit("", true);
+ assertGoodExit("", true, callback);
else
- assertBadExit("", true);
+ assertBadExit("", true, callback);
return result;
}
String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
}
String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
+ assertGoodExit(expectString, stringPresent, callback);
else
- assertBadExit(expectString, stringPresent);
+ assertBadExit(expectString, stringPresent, callback);
return result;
}
- void assertGoodExit(String s, boolean stringPresent) {
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
- assertEquals(0, shell.getExitCode());
+ if (0 != shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(0, shell.getExitCode());
+ }
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
- void assertBadExit(String s, boolean stringPresent) {
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
if (s.length() > 0)
@@ -409,7 +441,16 @@ public class ShellServerTest {
final String table = name.getMethodName();
// addauths
exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
+ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
+ public String getErrorMessage() {
+ try {
+ Connector c = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getConnector("root", new PasswordToken(secret));
+ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+ });
exec("addauths -s foo,bar", true);
exec("getauths", true, "foo", true);
exec("getauths", true, "bar", true);
@@ -486,10 +527,10 @@ public class ShellServerTest {
exec("flush -w");
exec("insert n 1 2 3");
exec("flush -w");
- oldCount = countFiles(tableId);
+ List<String> oldFiles = getFiles(tableId);
// at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
+ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
// compact some data:
exec("compact -b g -e z -w");
@@ -564,7 +605,8 @@ public class ShellServerTest {
exec("addsplits row5 row7");
make10();
exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
+ List<String> files = getFiles(tableId);
+ assertEquals("Found the following files: " + files, 3, files.size());
exec("deleterows -t " + table + " -b row5 -e row7", true);
assertEquals(2, countFiles(tableId));
exec("deletetable -f " + table);
@@ -767,12 +809,13 @@ public class ShellServerTest {
final String table = name.getMethodName();
exec("createtable " + table, true);
+
+ // Should be about a 3 second scan
+ for (int i = 0; i < 6; i++) {
+ exec("insert " + i + " cf cq value", true);
+ }
exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
+ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
Thread thread = new Thread() {
public void run() {
try {
@@ -788,14 +831,30 @@ public class ShellServerTest {
}
};
thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
+
+ List<String> scans = new ArrayList<String>();
+ // Try to find the active scan for about 5seconds
+ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
+ String currentScans = exec("listscans", true);
+ String[] lines = currentScans.split("\n");
+ for (int scanOffset = 2; i < lines.length; i++) {
+ String currentScan = lines[scanOffset];
+ if (currentScan.contains(table)) {
+ scans.add(currentScan);
+ }
+ }
+ UtilWaitThread.sleep(100);
+ }
thread.join();
+
+ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
+
+ for (String scan : scans) {
+ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
+ String parts[] = scan.split("\\|");
+ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
+ }
+
exec("deletetable -f " + table, true);
}
@@ -873,13 +932,24 @@ public class ShellServerTest {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
+
+ private List<String> getFiles(String tableId) throws IOException {
+ output.clear();
- private int countFiles(String tableId) throws IOException {
exec("scan -t !METADATA -np -c file -b " + tableId + " -e " + tableId + "~");
-
- log.debug("countFiles(): " + output.get());
-
- return output.get().split("\n").length - 1;
+
+ String[] lines = StringUtils.split(output.get(), "\n");
+ output.clear();
+
+ if (0 == lines.length) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
+ }
+
+ private int countFiles(String tableId) throws IOException {
+ return getFiles(tableId).size();
}
private String getTableId(String tableName) throws Exception {
[11/16] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --cc proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index abe540a,0000000..efe36b9
mode 100644,000000..100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@@ -1,1529 -1,0 +1,1567 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.proxy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.DevNull;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
+import org.apache.accumulo.proxy.thrift.AccumuloSecurityException;
+import org.apache.accumulo.proxy.thrift.ActiveCompaction;
+import org.apache.accumulo.proxy.thrift.ActiveScan;
+import org.apache.accumulo.proxy.thrift.BatchScanOptions;
+import org.apache.accumulo.proxy.thrift.Column;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.CompactionReason;
+import org.apache.accumulo.proxy.thrift.CompactionType;
+import org.apache.accumulo.proxy.thrift.Condition;
+import org.apache.accumulo.proxy.thrift.ConditionalStatus;
+import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
+import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
+import org.apache.accumulo.proxy.thrift.DiskUsage;
+import org.apache.accumulo.proxy.thrift.IteratorScope;
+import org.apache.accumulo.proxy.thrift.IteratorSetting;
+import org.apache.accumulo.proxy.thrift.Key;
+import org.apache.accumulo.proxy.thrift.KeyValue;
+import org.apache.accumulo.proxy.thrift.MutationsRejectedException;
+import org.apache.accumulo.proxy.thrift.PartialKey;
+import org.apache.accumulo.proxy.thrift.Range;
+import org.apache.accumulo.proxy.thrift.ScanColumn;
+import org.apache.accumulo.proxy.thrift.ScanOptions;
+import org.apache.accumulo.proxy.thrift.ScanResult;
+import org.apache.accumulo.proxy.thrift.ScanState;
+import org.apache.accumulo.proxy.thrift.ScanType;
+import org.apache.accumulo.proxy.thrift.SystemPermission;
+import org.apache.accumulo.proxy.thrift.TableExistsException;
+import org.apache.accumulo.proxy.thrift.TableNotFoundException;
+import org.apache.accumulo.proxy.thrift.TablePermission;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.accumulo.proxy.thrift.UnknownScanner;
+import org.apache.accumulo.proxy.thrift.UnknownWriter;
+import org.apache.accumulo.proxy.thrift.WriterOptions;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.test.functional.SlowIterator;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.server.TServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+/**
+ * Call every method on the proxy and try to verify that it works.
+ */
+public class SimpleProxyIT {
+
+ public static File macTestFolder = new File(System.getProperty("user.dir") + "/target/" + SimpleProxyIT.class.getName());
+
+ private static MiniAccumuloCluster accumulo;
+ private static String secret = "superSecret";
+ private static Random random = new Random();
+ private static TServer proxyServer;
+ private static Thread thread;
+ private static int proxyPort;
+ private static org.apache.accumulo.proxy.thrift.AccumuloProxy.Client client;
+ private static String principal = "root";
+
+ private static Map<String,String> properties = new TreeMap<String,String>() {
+ private static final long serialVersionUID = 1L;
+
+ {
+ put("password", secret);
+ }
+ };
+ private static ByteBuffer creds = null;
+
+ private static Class<? extends TProtocolFactory> protocolClass;
+
+ static Class<? extends TProtocolFactory> getRandomProtocol() {
+ List<Class<? extends TProtocolFactory>> protocolFactories = new ArrayList<Class<? extends TProtocolFactory>>();
+ protocolFactories.add(org.apache.thrift.protocol.TJSONProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TBinaryProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TTupleProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TCompactProtocol.Factory.class);
+
+ return protocolFactories.get(random.nextInt(protocolFactories.size()));
+ }
+
+ private static final AtomicInteger tableCounter = new AtomicInteger(0);
+
+ private static String makeTableName() {
+ return "test" + tableCounter.getAndIncrement();
+ }
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(macTestFolder);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+ FileUtils.deleteQuietly(macTestFolder);
+ macTestFolder.mkdirs();
+ MiniAccumuloConfig config = new MiniAccumuloConfig(macTestFolder, secret).setNumTservers(1);
+ accumulo = new MiniAccumuloCluster(config);
+ accumulo.start();
+
+ Properties props = new Properties();
+ props.put("instance", accumulo.getConfig().getInstanceName());
+ props.put("zookeepers", accumulo.getZooKeepers());
+ props.put("tokenClass", PasswordToken.class.getName());
+
+ protocolClass = getRandomProtocol();
+
+ proxyPort = PortUtils.getRandomFreePort();
+ proxyServer = Proxy.createProxyServer(org.apache.accumulo.proxy.thrift.AccumuloProxy.class, org.apache.accumulo.proxy.ProxyServer.class, proxyPort,
+ protocolClass, props);
+ thread = new Thread() {
+ @Override
+ public void run() {
+ proxyServer.serve();
+ }
+ };
+ thread.start();
+ while (!proxyServer.isServing())
+ UtilWaitThread.sleep(100);
+ client = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ creds = client.login(principal, properties);
+ }
+
+ @Test(timeout = 10000)
+ public void security() throws Exception {
+ client.createLocalUser(creds, "user", s2bb(secret));
+ ByteBuffer badLogin = client.login("user", properties);
+ client.dropLocalUser(creds, "user");
+ final String table = makeTableName();
+ client.createTable(creds, table, false, TimeType.MILLIS);
+
+ final IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ try {
+ client.addConstraint(badLogin, table, NumericValueConstraint.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.addSplits(badLogin, table, Collections.singleton(s2bb("1")));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.clearLocatorCache(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.compactTable(badLogin, table, null, null, null, true, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.cancelCompaction(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createTable(badLogin, table, false, TimeType.MILLIS);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.deleteTable(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.deleteRows(badLogin, table, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.tableExists(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.flushTable(badLogin, table, null, null, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getLocalityGroups(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getMaxRow(badLogin, table, Collections.<ByteBuffer> emptySet(), null, false, null, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getTableProperties(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listSplits(badLogin, table, 10000);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listTables(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.listConstraints(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.mergeTablets(badLogin, table, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.offlineTable(badLogin, table, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.onlineTable(badLogin, table, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeConstraint(badLogin, table, 0);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeTableProperty(badLogin, table, Property.TABLE_FILE_MAX.getKey());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.renameTable(badLogin, table, "someTableName");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ Map<String,Set<String>> groups = new HashMap<String,Set<String>>();
+ groups.put("group1", Collections.singleton("cf1"));
+ groups.put("group2", Collections.singleton("cf2"));
+ client.setLocalityGroups(badLogin, table, groups);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.setTableProperty(badLogin, table, Property.TABLE_FILE_MAX.getKey(), "0");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.tableIdMap(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.getSiteConfiguration(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getSystemConfiguration(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getTabletServers(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.getActiveScans(badLogin, "fake");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getActiveCompactions(badLogin, "fakse");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeProperty(badLogin, "table.split.threshold");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.setProperty(badLogin, "table.split.threshold", "500M");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.testClassLoad(badLogin, DevNull.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.authenticateUser(badLogin, "root", s2pp(secret));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ HashSet<ByteBuffer> auths = new HashSet<ByteBuffer>(Arrays.asList(s2bb("A"), s2bb("B")));
+ client.changeUserAuthorizations(badLogin, "stooge", auths);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.changeLocalUserPassword(badLogin, "stooge", s2bb(""));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createLocalUser(badLogin, "stooge", s2bb("password"));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.dropLocalUser(badLogin, "stooge");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getUserAuthorizations(badLogin, "stooge");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.grantSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.grantTablePermission(badLogin, "root", table, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.hasSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.hasTablePermission(badLogin, "root", table, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listLocalUsers(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.revokeSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.revokeTablePermission(badLogin, "root", table, TablePermission.ALTER_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createScanner(badLogin, table, new ScanOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createBatchScanner(badLogin, table, new BatchScanOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.updateAndFlush(badLogin, table, new HashMap<ByteBuffer,List<ColumnUpdate>>());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createWriter(badLogin, table, new WriterOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.attachIterator(badLogin, "slow", setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.checkIteratorConflicts(badLogin, table, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ final String TABLE_TEST = makeTableName();
+ client.cloneTable(badLogin, table, TABLE_TEST, false, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.exportTable(badLogin, table, "/tmp");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.importTable(badLogin, "testify", "/tmp");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getIteratorSetting(badLogin, table, "foo", IteratorScope.SCAN);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listIterators(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeIterator(badLogin, table, "name", EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.splitRangeByTablets(badLogin, table, client.getRowRange(ByteBuffer.wrap("row".getBytes())), 10);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ File importDir = tempFolder.newFolder("importDir");
+ File failuresDir = tempFolder.newFolder("failuresDir");
+ client.importDirectory(badLogin, table, importDir.getAbsolutePath(), failuresDir.getAbsolutePath(), true);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.pingTabletServer(badLogin, "fake");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.login("badUser", properties);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.testTableClassLoad(badLogin, table, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createConditionalWriter(badLogin, table, new ConditionalWriterOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ }
+
+ @Test(timeout = 10000)
+ public void tableNotFound() throws Exception {
+ final String doesNotExist = "doesNotExists";
+ try {
+ client.addConstraint(creds, doesNotExist, NumericValueConstraint.class.getName());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.addSplits(creds, doesNotExist, Collections.<ByteBuffer> emptySet());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ final IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+ try {
+ client.attachIterator(creds, doesNotExist, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.cancelCompaction(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.checkIteratorConflicts(creds, doesNotExist, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.clearLocatorCache(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ final String TABLE_TEST = makeTableName();
+ client.cloneTable(creds, doesNotExist, TABLE_TEST, false, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.compactTable(creds, doesNotExist, null, null, null, true, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createBatchScanner(creds, doesNotExist, new BatchScanOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createScanner(creds, doesNotExist, new ScanOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createWriter(creds, doesNotExist, new WriterOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.deleteRows(creds, doesNotExist, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.deleteTable(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.exportTable(creds, doesNotExist, "/tmp");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.flushTable(creds, doesNotExist, null, null, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getIteratorSetting(creds, doesNotExist, "foo", IteratorScope.SCAN);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getLocalityGroups(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getMaxRow(creds, doesNotExist, Collections.<ByteBuffer> emptySet(), null, false, null, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getTableProperties(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.grantTablePermission(creds, "root", doesNotExist, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.hasTablePermission(creds, "root", doesNotExist, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ File importDir = tempFolder.newFolder("importDir");
+ File failuresDir = tempFolder.newFolder("failuresDir");
+ client.importDirectory(creds, doesNotExist, importDir.getAbsolutePath(), failuresDir.getAbsolutePath(), true);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.listConstraints(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.listSplits(creds, doesNotExist, 10000);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.mergeTablets(creds, doesNotExist, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.offlineTable(creds, doesNotExist, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.onlineTable(creds, doesNotExist, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeConstraint(creds, doesNotExist, 0);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeIterator(creds, doesNotExist, "name", EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeTableProperty(creds, doesNotExist, Property.TABLE_FILE_MAX.getKey());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.renameTable(creds, doesNotExist, "someTableName");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.revokeTablePermission(creds, "root", doesNotExist, TablePermission.ALTER_TABLE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.setTableProperty(creds, doesNotExist, Property.TABLE_FILE_MAX.getKey(), "0");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.splitRangeByTablets(creds, doesNotExist, client.getRowRange(ByteBuffer.wrap("row".getBytes())), 10);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.updateAndFlush(creds, doesNotExist, new HashMap<ByteBuffer,List<ColumnUpdate>>());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getDiskUsage(creds, Collections.singleton(doesNotExist));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.testTableClassLoad(creds, doesNotExist, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createConditionalWriter(creds, doesNotExist, new ConditionalWriterOptions());
+ } catch (TableNotFoundException ex) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testExists() throws Exception {
+ client.createTable(creds, "ett1", false, TimeType.MILLIS);
+ client.createTable(creds, "ett2", false, TimeType.MILLIS);
+ try {
+ client.createTable(creds, "ett1", false, TimeType.MILLIS);
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ try {
+ client.renameTable(creds, "ett1", "ett2");
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ try {
+ client.cloneTable(creds, "ett1", "ett2", false, new HashMap<String,String>(), new HashSet<String>());
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testUnknownScanner() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ String scanner = client.createScanner(creds, TABLE_TEST, null);
+ assertFalse(client.hasNext(scanner));
+ client.closeScanner(scanner);
+
+ try {
+ client.hasNext(scanner);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+
+ try {
+ client.closeScanner(scanner);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+
+ try {
+ client.nextEntry("99999999");
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.nextK("99999999", 6);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.hasNext("99999999");
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.hasNext(UUID.randomUUID().toString());
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testUnknownWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ String writer = client.createWriter(creds, TABLE_TEST, null);
+ client.update(writer, mutation("row0", "cf", "cq", "value"));
+ client.flush(writer);
+ client.update(writer, mutation("row2", "cf", "cq", "value2"));
+ client.closeWriter(writer);
+
+ // this is a oneway call, so it does not throw exceptions
+ client.update(writer, mutation("row2", "cf", "cq", "value2"));
+
+ try {
+ client.flush(writer);
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.flush("99999");
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.flush(UUID.randomUUID().toString());
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.closeWriter("99999");
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testDelete() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row0", "cf", "cq", "value"));
+
+ assertScan(new String[][] {{"row0", "cf", "cq", "value"}}, TABLE_TEST);
+
+ ColumnUpdate upd = new ColumnUpdate(s2bb("cf"), s2bb("cq"));
+ upd.setDeleteCell(true);
+ Map<ByteBuffer,List<ColumnUpdate>> delete = Collections.singletonMap(s2bb("row0"), Collections.singletonList(upd));
+
+ client.updateAndFlush(creds, TABLE_TEST, delete);
+
+ assertScan(new String[][] {}, TABLE_TEST);
+ }
+
- @Test(timeout = 10000)
++ @Test(timeout = 60000)
+ public void testInstanceOperations() throws Exception {
+ int tservers = 0;
+ for (String tserver : client.getTabletServers(creds)) {
+ client.pingTabletServer(creds, tserver);
+ tservers++;
+ }
+ assertTrue(tservers > 0);
+
+ // get something we know is in the site config
+ Map<String,String> cfg = client.getSiteConfiguration(creds);
+ assertTrue(cfg.get("instance.dfs.dir").startsWith(macTestFolder.getPath()));
+
+ // set a property in zookeeper
+ client.setProperty(creds, "table.split.threshold", "500M");
+
+ // check that we can read it
+ for (int i = 0; i < 5; i++) {
+ cfg = client.getSystemConfiguration(creds);
+ if ("500M".equals(cfg.get("table.split.threshold")))
+ break;
+ UtilWaitThread.sleep(200);
+ }
+ assertEquals("500M", cfg.get("table.split.threshold"));
+
+ // unset the setting, check that it's not what it was
+ client.removeProperty(creds, "table.split.threshold");
+ for (int i = 0; i < 5; i++) {
+ cfg = client.getSystemConfiguration(creds);
+ if (!"500M".equals(cfg.get("table.split.threshold")))
+ break;
+ UtilWaitThread.sleep(200);
+ }
+ assertNotEquals("500M", cfg.get("table.split.threshold"));
+
+ // try to load some classes via the proxy
+ assertTrue(client.testClassLoad(creds, DevNull.class.getName(), SortedKeyValueIterator.class.getName()));
+ assertFalse(client.testClassLoad(creds, "foo.bar", SortedKeyValueIterator.class.getName()));
+
+ // create a table that's very slow, so we can look for scans/compactions
+ client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
++
++ // Should take 5 seconds to read every record
++ for (int i = 0; i < 20; i++) {
++ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
++ }
++
++ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
+ client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
+
+ // scan
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ String scanner;
+ try {
+ Client client2 = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ scanner = client2.createScanner(creds, "slow", null);
+ client2.nextK(scanner, 10);
+ client2.closeScanner(scanner);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ t.start();
++
+ // look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
++ List<ActiveScan> scans = new ArrayList<ActiveScan>();
++ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
+ for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
++ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
++ for (ActiveScan scan : scansForServer) {
++ if ("root".equals(scan.getUser())) {
++ scans.add(scan);
++ }
++ }
++
+ if (!scans.isEmpty())
- break loop;
++ break;
+ UtilWaitThread.sleep(10);
+ }
+ }
+ t.join();
++
+ assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
++ boolean found = false;
++ Map<String,String> map = null;
++ for (int i = 0; i < scans.size() && !found; i++) {
++ ActiveScan scan = scans.get(i);
++ if ("root".equals(scan.getUser())) {
++ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
++ assertEquals(ScanType.SINGLE, scan.getType());
++ assertEquals("slow", scan.getTable());
++
++ map = client.tableIdMap(creds);
++ assertEquals(map.get("slow"), scan.getExtent().tableId);
++ assertTrue(scan.getExtent().endRow == null);
++ assertTrue(scan.getExtent().prevEndRow == null);
++ found = true;
++ }
++ }
++
++ assertTrue("Could not find a scan against the 'slow' table", found);
+
+ // start a compaction
+ t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Client client2 = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ client2.compactTable(creds, "slow", null, null, null, true, true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ t.start();
+
++ final String desiredTableId = map.get("slow");
++
+ // try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
++ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
++ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
++ // Iterate over the tservers
+ for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
++ // And get the compactions on each
++ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
++ for (ActiveCompaction compact : compactionsOnServer) {
++ // There might be other compactions occurring (e.g. on METADATA) in which
++ // case we want to prune out those that aren't for our slow table
++ if (desiredTableId.equals(compact.getExtent().tableId)) {
++ compactions.add(compact);
++ }
++ }
++
++ // If we found a compaction for the table we wanted, so we can stop looking
+ if (!compactions.isEmpty())
- break loop2;
++ break;
+ }
+ UtilWaitThread.sleep(10);
+ }
+ t.join();
++
+ // verify the compaction information
+ assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
++ for (ActiveCompaction c : compactions) {
++ if (desiredTableId.equals(c.getExtent().tableId)) {
++ assertTrue(c.inputFiles.isEmpty());
++ assertEquals(CompactionType.MINOR, c.getType());
++ assertEquals(CompactionReason.USER, c.getReason());
++ assertEquals("", c.localityGroup);
++ assertTrue(c.outputFile.contains("default_tablet"));
++
++ return;
++ }
++ }
++ fail("Expection to find running compaction for table 'slow' but did not find one");
+ }
+
+ @Test
+ public void testSecurityOperations() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ // check password
+ assertTrue(client.authenticateUser(creds, "root", s2pp(secret)));
+ assertFalse(client.authenticateUser(creds, "root", s2pp("")));
+
+ // create a user
+ client.createLocalUser(creds, "stooge", s2bb("password"));
+ // change auths
+ Set<String> users = client.listLocalUsers(creds);
+ assertEquals(new HashSet<String>(Arrays.asList("root", "stooge")), users);
+ HashSet<ByteBuffer> auths = new HashSet<ByteBuffer>(Arrays.asList(s2bb("A"), s2bb("B")));
+ client.changeUserAuthorizations(creds, "stooge", auths);
+ List<ByteBuffer> update = client.getUserAuthorizations(creds, "stooge");
+ assertEquals(auths, new HashSet<ByteBuffer>(update));
+
+ // change password
+ client.changeLocalUserPassword(creds, "stooge", s2bb(""));
+ assertTrue(client.authenticateUser(creds, "stooge", s2pp("")));
+
+ // check permission failure
+ @SuppressWarnings("serial")
+ ByteBuffer stooge = client.login("stooge", new TreeMap<String,String>() {
+ {
+ put("password", "");
+ }
+ });
+
+ try {
+ client.createTable(stooge, "fail", true, TimeType.MILLIS);
+ fail("should not create the table");
+ } catch (AccumuloSecurityException ex) {
+ assertFalse(client.listTables(creds).contains("fail"));
+ }
+ // grant permissions and test
+ assertFalse(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ client.grantSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE);
+ assertTrue(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ client.createTable(stooge, "success", true, TimeType.MILLIS);
+ client.listTables(creds).contains("succcess");
+
+ // revoke permissions
+ client.revokeSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE);
+ assertFalse(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ try {
+ client.createTable(stooge, "fail", true, TimeType.MILLIS);
+ fail("should not create the table");
+ } catch (AccumuloSecurityException ex) {
+ assertFalse(client.listTables(creds).contains("fail"));
+ }
+ // create a table to test table permissions
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ // denied!
+ try {
+ String scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 100);
+ fail("stooge should not read table test");
+ } catch (AccumuloSecurityException ex) {}
+ // grant
+ assertFalse(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ client.grantTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ);
+ assertTrue(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ String scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 10);
+ client.closeScanner(scanner);
+ // revoke
+ client.revokeTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ);
+ assertFalse(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ try {
+ scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 100);
+ fail("stooge should not read table test");
+ } catch (AccumuloSecurityException ex) {}
+
+ // delete user
+ client.dropLocalUser(creds, "stooge");
+ users = client.listLocalUsers(creds);
+ assertEquals(1, users.size());
+
+ }
+
+ @Test
+ public void testBatchWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+
+ WriterOptions writerOptions = new WriterOptions();
+ writerOptions.setLatencyMs(10000);
+ writerOptions.setMaxMemory(2);
+ writerOptions.setThreads(1);
+ writerOptions.setTimeoutMs(100000);
+
+ String batchWriter = client.createWriter(creds, TABLE_TEST, writerOptions);
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ try {
+ client.flush(batchWriter);
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException ex) {}
+ try {
+ client.closeWriter(batchWriter);
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException e) {}
+
+ client.removeConstraint(creds, TABLE_TEST, 2);
+
+ assertScan(new String[][] {}, TABLE_TEST);
+
+ UtilWaitThread.sleep(2000);
+
+ writerOptions = new WriterOptions();
+ writerOptions.setLatencyMs(10000);
+ writerOptions.setMaxMemory(3000);
+ writerOptions.setThreads(1);
+ writerOptions.setTimeoutMs(100000);
+
+ batchWriter = client.createWriter(creds, TABLE_TEST, writerOptions);
+
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ client.flush(batchWriter);
+ client.closeWriter(batchWriter);
+
+ assertScan(new String[][] {{"row1", "cf", "cq", "x"}}, TABLE_TEST);
+
+ client.deleteTable(creds, TABLE_TEST);
+ }
+
+ @Test
+ public void testTableOperations() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ // constraints
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+ assertEquals(2, client.listConstraints(creds, TABLE_TEST).size());
+
+ UtilWaitThread.sleep(2000);
+
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "123"));
+
+ try {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException ex) {}
+
+ client.removeConstraint(creds, TABLE_TEST, 2);
+
+ UtilWaitThread.sleep(2000);
+
+ assertEquals(1, client.listConstraints(creds, TABLE_TEST).size());
+
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
+ assertScan(new String[][] {{"row1", "cf", "cq", "x"}}, TABLE_TEST);
+ // splits, merge
+ client.addSplits(creds, TABLE_TEST, new HashSet<ByteBuffer>(Arrays.asList(s2bb("a"), s2bb("m"), s2bb("z"))));
+ List<ByteBuffer> splits = client.listSplits(creds, TABLE_TEST, 1);
+ assertEquals(Arrays.asList(s2bb("m")), splits);
+ client.mergeTablets(creds, TABLE_TEST, null, s2bb("m"));
+ splits = client.listSplits(creds, TABLE_TEST, 10);
+ assertEquals(Arrays.asList(s2bb("m"), s2bb("z")), splits);
+ client.mergeTablets(creds, TABLE_TEST, null, null);
+ splits = client.listSplits(creds, TABLE_TEST, 10);
+ List<ByteBuffer> empty = Collections.emptyList();
+ assertEquals(empty, splits);
+ // iterators
+ client.deleteTable(creds, TABLE_TEST);
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ HashMap<String,String> options = new HashMap<String,String>();
+ options.put("type", "STRING");
+ options.put("columns", "cf");
+ IteratorSetting setting = new IteratorSetting(10, TABLE_TEST, SummingCombiner.class.getName(), options);
+ client.attachIterator(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
+ for (int i = 0; i < 10; i++) {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "1"));
+ }
+ assertScan(new String[][] {{"row1", "cf", "cq", "10"}}, TABLE_TEST);
+ try {
+ client.checkIteratorConflicts(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
+ fail("checkIteratorConflicts did not throw an exception");
+ } catch (Exception ex) {}
+ client.deleteRows(creds, TABLE_TEST, null, null);
+ client.removeIterator(creds, TABLE_TEST, "test", EnumSet.allOf(IteratorScope.class));
+ String expected[][] = new String[10][];
+ for (int i = 0; i < 10; i++) {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row" + i, "cf", "cq", "" + i));
+ expected[i] = new String[] {"row" + i, "cf", "cq", "" + i};
+ client.flushTable(creds, TABLE_TEST, null, null, true);
+ }
+ assertScan(expected, TABLE_TEST);
+ // clone
+ final String TABLE_TEST2 = makeTableName();
+ client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
+ assertScan(expected, TABLE_TEST2);
+ client.deleteTable(creds, TABLE_TEST2);
+
+ // don't know how to test this, call it just for fun
+ client.clearLocatorCache(creds, TABLE_TEST);
+
+ // compact
+ client.compactTable(creds, TABLE_TEST, null, null, null, true, true);
+ assertEquals(1, countFiles(TABLE_TEST));
+ assertScan(expected, TABLE_TEST);
+
+ // get disk usage
+ client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
+ Set<String> tablesToScan = new HashSet<String>();
+ tablesToScan.add(TABLE_TEST);
+ tablesToScan.add(TABLE_TEST2);
+ tablesToScan.add("foo");
+ client.createTable(creds, "foo", true, TimeType.MILLIS);
+ List<DiskUsage> diskUsage = (client.getDiskUsage(creds, tablesToScan));
+ assertEquals(2, diskUsage.size());
+ assertEquals(1, diskUsage.get(0).getTables().size());
+ assertEquals(2, diskUsage.get(1).getTables().size());
+ client.compactTable(creds, TABLE_TEST2, null, null, null, true, true);
+ diskUsage = (client.getDiskUsage(creds, tablesToScan));
+ assertEquals(3, diskUsage.size());
+ assertEquals(1, diskUsage.get(0).getTables().size());
+ assertEquals(1, diskUsage.get(1).getTables().size());
+ assertEquals(1, diskUsage.get(2).getTables().size());
+ client.deleteTable(creds, "foo");
+ client.deleteTable(creds, TABLE_TEST2);
+
+ // export/import
+ File dir = tempFolder.newFolder("test");
+ File destDir = tempFolder.newFolder("test_dest");
+ client.offlineTable(creds, TABLE_TEST, false);
+ client.exportTable(creds, TABLE_TEST, dir.getAbsolutePath());
+ // copy files to a new location
+ FileSystem fs = FileSystem.get(new Configuration());
+ FSDataInputStream is = fs.open(new Path(dir + "/distcp.txt"));
+ BufferedReader r = new BufferedReader(new InputStreamReader(is));
+ while (true) {
+ String line = r.readLine();
+ if (line == null)
+ break;
+ Path srcPath = new Path(line);
+ FileUtils.copyFile(new File(srcPath.toUri().getPath()), new File(destDir, srcPath.getName()));
+ }
+ client.deleteTable(creds, TABLE_TEST);
+ client.importTable(creds, "testify", destDir.getAbsolutePath());
+ assertScan(expected, "testify");
+ client.deleteTable(creds, "testify");
+
+ try {
+ // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files
+ client.importTable(creds, "testify2", destDir.getAbsolutePath());
+ fail();
+ } catch (Exception e) {}
+
+ assertFalse(client.listTables(creds).contains("testify2"));
+
+ // Locality groups
+ client.createTable(creds, "test", true, TimeType.MILLIS);
+ Map<String,Set<String>> groups = new HashMap<String,Set<String>>();
+ groups.put("group1", Collections.singleton("cf1"));
+ groups.put("group2", Collections.singleton("cf2"));
+ client.setLocalityGroups(creds, "test", groups);
+ assertEquals(groups, client.getLocalityGroups(creds, "test"));
+ // table properties
+ Map<String,String> orig = client.getTableProperties(creds, "test");
+ client.setTableProperty(creds, "test", "table.split.threshold", "500M");
+ Map<String,String> update = client.getTableProperties(creds, "test");
+ assertEquals(update.get("table.split.threshold"), "500M");
+ client.removeTableProperty(creds, "test", "table.split.threshold");
+ update = client.getTableProperties(creds, "test");
+ assertEquals(orig, update);
+ // rename table
+ Map<String,String> tables = client.tableIdMap(creds);
+ client.renameTable(creds, "test", "bar");
+ Map<String,String> tables2 = client.tableIdMap(creds);
+ assertEquals(tables.get("test"), tables2.get("bar"));
+ // table exists
+ assertTrue(client.tableExists(creds, "bar"));
+ assertFalse(client.tableExists(creds, "test"));
+ // bulk import
+ String filename = dir + "/bulk/import/rfile.rf";
+ FileSKVWriter writer = FileOperations.getInstance().openWriter(filename, fs, fs.getConf(), DefaultConfiguration.getInstance());
+ writer.startDefaultLocalityGroup();
+ writer.append(new org.apache.accumulo.core.data.Key(new Text("a"), new Text("b"), new Text("c")), new Value("value".getBytes()));
+ writer.close();
+ fs.mkdirs(new Path(dir + "/bulk/fail"));
+ client.importDirectory(creds, "bar", dir + "/bulk/import", dir + "/bulk/fail", true);
+ String scanner = client.createScanner(creds, "bar", null);
+ ScanResult more = client.nextK(scanner, 100);
+ client.closeScanner(scanner);
+ assertEquals(1, more.results.size());
+ ByteBuffer maxRow = client.getMaxRow(creds, "bar", null, null, false, null, false);
+ assertEquals(s2bb("a"), maxRow);
+
+ assertFalse(client.testTableClassLoad(creds, "bar", "abc123", SortedKeyValueIterator.class.getName()));
+ assertTrue(client.testTableClassLoad(creds, "bar", VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
+ }
+
+ private Condition newCondition(String cf, String cq) {
+ return new Condition(new Column(s2bb(cf), s2bb(cq), s2bb("")));
+ }
+
+ private Condition newCondition(String cf, String cq, String val) {
+ return newCondition(cf, cq).setValue(s2bb(val));
+ }
+
+ private Condition newCondition(String cf, String cq, long ts, String val) {
+ return newCondition(cf, cq).setValue(s2bb(val)).setTimestamp(ts);
+ }
+
+ private ColumnUpdate newColUpdate(String cf, String cq, String val) {
+ return new ColumnUpdate(s2bb(cf), s2bb(cq)).setValue(s2bb(val));
+ }
+
+ private ColumnUpdate newColUpdate(String cf, String cq, long ts, String val) {
+ return new ColumnUpdate(s2bb(cf), s2bb(cq)).setTimestamp(ts).setValue(s2bb(val));
+ }
+
+ private void assertScan(String[][] expected, String table) throws Exception {
+ String scid = client.createScanner(creds, table, new ScanOptions());
+ ScanResult keyValues = client.nextK(scid, expected.length + 1);
+
+ assertEquals(expected.length, keyValues.results.size());
+ assertFalse(keyValues.more);
+
+ for (int i = 0; i < keyValues.results.size(); i++) {
+ checkKey(expected[i][0], expected[i][1], expected[i][2], expected[i][3], keyValues.results.get(i));
+ }
+
+ client.closeScanner(scid);
+ }
+
+ @Test
+ public void testConditionalWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+
+ String cwid = client.createConditionalWriter(creds, TABLE_TEST, new ConditionalWriterOptions());
+
+ Map<ByteBuffer,ConditionalUpdates> updates = new HashMap<ByteBuffer,ConditionalUpdates>();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", 10, "1"),
+ newColUpdate("data", "img", "73435435"))));
+
+ Map<ByteBuffer,ConditionalStatus> results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "73435435"}, {"00345", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test not setting values on conditions
+ updates.clear();
+
+ updates.put(s2bb("00345"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", "2"))));
+ updates.put(s2bb("00346"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", "1"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00345")));
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00346")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "73435435"}, {"00345", "meta", "seq", "1"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting values on conditions
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", "1")), Arrays.asList(newColUpdate("meta", "seq", 20, "2"),
+ newColUpdate("data", "img", "567890"))));
+
+ updates.put(s2bb("00346"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", "2")), Arrays.asList(newColUpdate("meta", "seq", "3"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00346")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "567890"}, {"00345", "meta", "seq", "2"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting timestamp on condition to a non-existant version
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", 10, "2")), Arrays.asList(newColUpdate("meta", "seq", 30, "3"),
+ newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "567890"}, {"00345", "meta", "seq", "2"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting timestamp to an existing version
+
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", 20, "2")), Arrays.asList(newColUpdate("meta", "seq", 30, "3"),
+ newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // run test w/ condition that has iterators
+ // following should fail w/o iterator
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+
+ updates.clear();
+ updates.put(s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "count", "3")), Arrays.asList(newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}}, TABLE_TEST);
+
+ // following test w/ iterator setup should succeed
+ Condition iterCond = newCondition("data", "count", "3");
+ Map<String,String> props = new HashMap<String,String>();
+ props.put("type", "STRING");
+ props.put("columns", "data:count");
+ IteratorSetting is = new IteratorSetting(1, "sumc", SummingCombiner.class.getName(), props);
+ iterCond.setIterators(Arrays.asList(is));
+
+ updates.clear();
+ updates.put(s2bb("00347"), new ConditionalUpdates(Arrays.asList(iterCond), Arrays.asList(newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // test a mutation that violated a constraint
+ updates.clear();
+ updates.put(s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890")), Arrays.asList(newColUpdate("data", "count", "A"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.VIOLATED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // run test with two conditions
+ // both conditions should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "565"), newCondition("data", "count", "2")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // one condition should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890"), newCondition("data", "count", "2")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // one condition should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "565"), newCondition("data", "count", "1")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // both conditions should succeed
+
+ ConditionalStatus result = client.updateRowConditionally(
+ creds,
+ TABLE_TEST,
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890"), newCondition("data", "count", "1")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ assertEquals(ConditionalStatus.ACCEPTED, result);
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}}, TABLE_TEST);
+
+ client.closeConditionalWriter(cwid);
+ try {
+ client.updateRowsConditionally(cwid, updates);
+ fail("conditional writer not closed");
+ } catch (UnknownWriter uk) {}
+
+ // run test with colvis
+ client.createLocalUser(creds, "cwuser", s2bb("bestpasswordever"));
+ client.changeUserAuthorizations(creds, "cwuser", Collections.singleton(s2bb("A")));
+ client.grantTablePermission(creds, "cwuser", TABLE_TEST, TablePermission.WRITE);
+ client.grantTablePermission(creds, "cwuser", TABLE_TEST, TablePermission.READ);
+
+ ByteBuffer cwuCreds = client.login("cwuser", Collections.singletonMap("password", "bestpasswordever"));
+
+ cwid = client.createConditionalWriter(cwuCreds, TABLE_TEST, new ConditionalWriterOptions().setAuthorizations(Collections.singleton(s2bb("A"))));
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A")))), Arrays.asList(newColUpdate("data", "seq", "1"),
+ newColUpdate("data", "c", "1").setColVisibility(s2bb("A")))));
+ updates.put(s2bb("00349"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("B")))), Arrays.asList(newColUpdate("data", "seq", "1"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00348")));
+ assertEquals(ConditionalStatus.INVISIBLE_VISIBILITY, results.get(s2bb("00349")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "1"}}, TABLE_TEST);
+
+ updates.clear();
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A"))).setValue(s2bb("0"))), Arrays.asList(
+ newColUpdate("data", "seq", "2"), newColUpdate("data", "c", "2").setColVisibility(s2bb("A")))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00348")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "1"}}, TABLE_TEST);
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A"))).setValue(s2bb("1"))), Arrays.asList(
+ newColUpdate("data", "seq", "2"), newColUpdate("data", "c", "2").setColVisibility(s2bb("A")))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00348")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "2"}}, TABLE_TEST);
+
+ client.closeConditionalWriter(cwid);
+ try {
+ client.updateRowsConditionally(cwid, updates);
+ fail("conditional writer not closed");
+ } catch (UnknownWriter uk) {}
+
+ client.dropLocalUser(creds, "cwuser");
+
+ }
+
+ private void checkKey(String row, String cf, String cq, String val, KeyValue keyValue) {
+ assertEquals(row, ByteBufferUtil.toString(keyValue.key.row));
+ assertEquals(cf, ByteBufferUtil.toString(keyValue.key.colFamily));
+ assertEquals(cq, ByteBufferUtil.toString(keyValue.key.colQualifier));
+ assertEquals("", ByteBufferUtil.toString(keyValue.key.colVisibility));
+ assertEquals(val, ByteBufferUtil.toString(keyValue.value));
+ }
+
+ // scan metadata for file entries for the given table
+ private int countFiles(String table) throws Exception {
+ Map<String,String> tableIdMap = client.tableIdMap(creds);
+ String tableId = tableIdMap.get(table);
+ Key start = new Key();
+ start.row = s2bb(tableId + ";");
+ Key end = new Key();
+ end.row = s2bb(tableId + "<");
+ end = client.getFollowing(end, PartialKey.ROW);
+ ScanOptions opt = new ScanOptions();
+ opt.range = new Range(start, true, end, false);
+ opt.columns = Collections.singletonList(new ScanColumn(s2bb("file")));
+ String scanner = client.createScanner(creds, MetadataTable.NAME, opt);
+ int result = 0;
+ while (true) {
+ ScanResult more = client.nextK(scanner, 100);
+ result += more.getResults().size();
+ if (!more.more)
+ break;
+ }
+ return result;
+ }
+
+ private Map<ByteBuffer,List<ColumnUpdate>> mutation(String row, String cf, String cq, String value) {
+ ColumnUpdate upd = new ColumnUpdate(s2bb(cf), s2bb(cq));
+ upd.setValue(value.getBytes());
+ return Collections.singletonMap(s2bb(row), Collections.singletonList(upd));
+ }
+
+ private ByteBuffer s2bb(String cf) {
+ return ByteBuffer.wrap(cf.getBytes());
+ }
+
+ private Map<String,String> s2pp(String cf) {
+ Map<String,String> toRet = new TreeMap<String,String>();
+ toRet.put("password", cf);
+ return toRet;
+ }
+
+ static private ByteBuffer t2bb(Text t) {
+ return ByteBuffer.wrap(t.getBytes());
+ }
+
+ @Test
+ public void testGetRowRange() throws Exception {
+ Range range = client.getRowRange(s2bb("xyzzy"));
+ org.apache.accumulo.core.data.Range range2 = new org.apache.accumulo.core.data.Range(new Text("xyzzy"));
+ assertEquals(0, range.start.row.compareTo(t2bb(range2.getStartKey().getRow())));
+ assertEquals(0, range.stop.row.compareTo(t2bb(range2.getEndKey().getRow())));
+ assertEquals(range.startInclusive, range2.isStartKeyInclusive());
+ assertEquals(range.stopInclusive, range2.isEndKeyInclusive());
+ assertEquals(0, range.start.colFamily.compareTo(t2bb(range2.getStartKey().getColumnFamily())));
+ assertEquals(0, range.start.colQualifier.compareTo(t2bb(range2.getStartKey().getColumnQualifier())));
+ assertEquals(0, range.stop.colFamily.compareTo(t2bb(range2.getEndKey().getColumnFamily())));
+ assertEquals(0, range.stop.colQualifier.compareTo(t2bb(range2.getEndKey().getColumnQualifier())));
+ assertEquals(range.start.timestamp, range.start.timestamp);
+ assertEquals(range.stop.timestamp, range.stop.timestamp);
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ }
+}
[13/16] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index 7a1121f,0000000..e0ad1bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@@ -1,1159 -1,0 +1,1238 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
++import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
++import java.util.ArrayList;
++import java.util.Arrays;
++import java.util.Collections;
++import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.commons.io.FileUtils;
++import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class ShellServerIT extends SimpleMacIT {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static final Logger log = Logger.getLogger(ShellServerIT.class);
+
+ public static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() throws IOException {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+
++ private static abstract class ErrorMessageCallback {
++ public abstract String getErrorMessage();
++ }
++
++ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
++ private static final String empty = "";
++ public String getErrorMessage() {
++ return empty;
++ }
++ }
++
++ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
++
+ public TestOutputStream output;
+ public StringInputStream input;
+ public Shell shell;
+ private static Process traceProcess;
+
+ @Rule
+ public TestName name = new TestName();
+
+ String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ String exec(String cmd, boolean expectGoodExit) throws IOException {
++ return exec(cmd, expectGoodExit, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
- assertGoodExit("", true);
++ assertGoodExit("", true, callback);
+ else
- assertBadExit("", true);
++ assertBadExit("", true, callback);
+ return result;
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
++ return exec(cmd, expectGoodExit, expectString, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
++ return exec(cmd, expectGoodExit, expectString, true, callback);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
++ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
++ assertGoodExit(expectString, stringPresent, callback);
+ else
- assertBadExit(expectString, stringPresent);
++ assertBadExit(expectString, stringPresent, callback);
+ return result;
+ }
+
+ void assertGoodExit(String s, boolean stringPresent) {
++ assertGoodExit(s, stringPresent, noop);
++ }
++
++ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.info(output.get());
- assertEquals(0, shell.getExitCode());
++ if (0 != shell.getExitCode()) {
++ String errorMsg = callback.getErrorMessage();
++ assertEquals(errorMsg, 0, shell.getExitCode());
++ }
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
- void assertBadExit(String s, boolean stringPresent) {
++ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.debug(output.get());
+ assertTrue(shell.getExitCode() > 0);
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // history file is updated in $HOME
+ System.setProperty("HOME", getFolder().getAbsolutePath());
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
+ // use reflection to call this method so it does not need to be made public
+ Method method = getStaticCluster().getClass().getDeclaredMethod("exec", Class.class, String[].class);
+ method.setAccessible(true);
+ traceProcess = (Process) method.invoke(getStaticCluster(), TraceServer.class, new String[0]);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @Before
+ public void setupShell() throws Exception {
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
- shell = new Shell(new ConsoleReader(input, output));
++ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
++ shell = new Shell(new ConsoleReader(input, output), pw);
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(),
+ "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ traceProcess.destroy();
+ }
+
+ @After
+ public void deleteTables() throws Exception {
+ Connector c = getConnector();
+ for (String table : c.tableOperations().list()) {
+ if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace"))
+ try {
+ c.tableOperations().delete(table);
+ } catch (TableNotFoundException e) {
+ // don't care
+ }
+ }
+ }
+
+ @After
+ public void tearDownShell() {
+ shell.shutdown();
+ }
+
+ @Test(timeout = 60000)
+ public void exporttableImporttable() throws Exception {
+ final String table = name.getMethodName(), table2 = table + "2";
+
+ // exporttable / importtable
+ exec("createtable " + table + " -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t " + table + " -s table.split.threshold=345M", true);
+ exec("offline " + table, true);
+ String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString();
+ exec("exporttable -t " + table + " " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable " + table2 + " " + import_, true);
+ exec("config -t " + table2 + " -np", true, "345M", true);
+ exec("getsplits -t " + table2, true, "row5", true);
+ exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true);
+ exec("onlinetable " + table, true);
+ exec("deletetable -f " + table, true);
+ exec("deletetable -f " + table2, true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
+ @Test(timeout = 45000)
+ public void setscaniterDeletescaniter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setscaniter, deletescaniter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING");
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder());
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
+ @Test(timeout = 45000)
+ public void egrep() throws Exception {
+ final String table = name.getMethodName();
+
+ // egrep
+ exec("createtable " + table);
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void du() throws Exception {
+ final String table = name.getMethodName();
+
+ // create and delete a table so we get out of a table context in the shell
+ exec("notable", true);
+
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
+
+ output.clear();
+ exec("createtable " + table);
+ make10();
+ exec("flush -t " + table + " -w");
+ exec("du " + table, true, " [" + table + "]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*[1-9][0-9][0-9]\\s\\[" + table + "\\]\\n"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
+ @Test(timeout = 45000)
+ public void user() throws Exception {
+ final String table = name.getMethodName();
+
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ input.set("secret\nsecret\n");
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (" + MetadataTable.NAME + "): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ input.set("secret\nsecret\n");
+ exec("user xyzzy", true);
+ exec("createtable " + table, true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f " + table, true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
+ @Test(timeout = 45000)
+ public void iter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n name", false);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f " + table);
+ // list iter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n name", false);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void setIterOptionPrompt() throws Exception {
+ Connector conn = getConnector();
+ String tableName = name.getMethodName();
+
+ exec("createtable " + tableName);
+ input.set("\n\n");
+ // Setting a non-optiondescriber with no name should fail
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", false);
+
+ // Name as option will work
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+
+ String expectedKey = "table.iterator.scan.cfcounter";
+ String expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ TableOperations tops = conn.tableOperations();
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\nname1 value1\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name1";
+ expectedValue = "value1";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("\nname1 value1.1,value1.2,value1.3\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+ expectedKey = "table.iterator.scan.cfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name1";
+ expectedValue = "value1.1,value1.2,value1.3";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ }
+
+ protected void checkTableForProperty(TableOperations tops, String tableName, String expectedKey, String expectedValue) throws Exception {
+ for (int i = 0; i < 5; i++) {
+ for (Entry<String,String> entry : tops.getProperties(tableName)) {
+ if (expectedKey.equals(entry.getKey())) {
+ assertEquals(expectedValue, entry.getValue());
+ return;
+ }
+ }
+ Thread.sleep(500);
+ }
+
+ fail("Failed to find expected property on " + tableName + ": " + expectedKey + "=" + expectedValue);
+ }
+
+ @Test(timeout = 45000)
+ public void notable() throws Exception {
+ final String table = name.getMethodName();
+
+ // notable
+ exec("createtable " + table, true);
+ exec("scan", true, " " + table + ">", true);
+ assertTrue(output.get().contains(" " + table + ">"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" " + table + ">"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue("Diff was actually " + diff, diff >= 200);
+ assertTrue("Diff was actually " + diff, diff < 600);
+ }
+
+ @Test(timeout = 45000)
+ public void addauths() throws Exception {
+ final String table = name.getMethodName();
+ // addauths
+ exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
++ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
++ public String getErrorMessage() {
++ try {
++ Connector c = getConnector();
++ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
++ } catch (Exception e) {
++ return "Could not check authorizations";
++ }
++ }
++ });
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
+ @Test(timeout = 45000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ if (shell.getReader().getTerminal().isAnsiSupported()) {
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ } else {
+ exec("cls", false, "does not support");
+ exec("clear", false, "does not support");
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void clonetable() throws Exception {
+ final String table = name.getMethodName(), clone = table + "_clone";
+
+ // clonetable
+ exec("createtable " + table + " -evc");
+ exec("config -t " + table + " -s table.split.threshold=123M", true);
+ exec("addsplits -t " + table + " a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable " + table + " " + clone);
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t " + clone, true, "VisibilityConstraint=2", true);
+ exec("config -t " + clone + " -np", true, "123M", true);
+ exec("getsplits -t " + clone, true, "a\nb\nc\n");
+ exec("deletetable -f " + table);
+ exec("deletetable -f " + clone);
+ }
+
+ @Test(timeout = 45000)
+ public void testCompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ // compact
+ exec("createtable " + table);
+
+ String tableId = getTableId(table);
+
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles(tableId);
+ // merge two files into one
+ exec("compact -t " + table + " -w");
+ assertTrue(countFiles(tableId) < oldCount);
+ exec("addsplits -t " + table + " f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
- oldCount = countFiles(tableId);
++ List<String> oldFiles = getFiles(tableId);
+
+ // at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
++ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
+
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertEquals(2, countFiles(tableId));
+ exec("compact -w");
+ assertEquals(2, countFiles(tableId));
+ exec("merge --all -t " + table);
+ exec("compact -w");
+ assertEquals(1, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void constraint() throws Exception {
+ final String table = name.getMethodName();
+
+ // constraint
+ exec("constraint -l -t " + MetadataTable.NAME + "", true, "MetadataConstraints=1", true);
+ exec("createtable " + table + " -evc");
+
+ // Make sure the table is fully propagated through zoocache
+ getTableId(table);
+
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", true);
+ exec("constraint -t " + table + " -d 2", true, "Removed constraint 2 from table " + table);
+ // wait for zookeeper updates to propagate
+ UtilWaitThread.sleep(1000);
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deletemany() throws Exception {
+ final String table = name.getMethodName();
+
+ // deletemany
+ exec("createtable " + table);
+ make10();
+ assertEquals(10, countkeys(table));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys(table));
+ exec("scan -t " + table + " -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys(table));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deleterows() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ final String tableId = getTableId(table);
+
+ // deleterows
+ int base = countFiles(tableId);
+ assertEquals(0, base);
+
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
++ List<String> files = getFiles(tableId);
++ assertEquals("Found the following files: " + files, 3, files.size());
+ exec("deleterows -t " + table + " -b row5 -e row7", true);
+ assertEquals(2, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void groups() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("setgroups -t " + table + " alpha=a,b,c num=3,2,1");
+ exec("getgroups -t " + table, true, "alpha=a,b,c", true);
+ exec("getgroups -t " + table, true, "num=1,2,3", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void grep() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
+
+ // @Test(timeout = 45000)
+ public void history() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("history -c", true);
+ exec("createtable " + table);
+ exec("deletetable -f " + table);
+ exec("history", true, table, true);
+ exec("history", true, "history", true);
+ }
+
+ @Test(timeout = 45000)
+ public void importDirectory() throws Exception {
+ final String table = name.getMethodName();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = new File(getFolder(), "import");
+ importDir.mkdir();
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = new File(getFolder(), "errors");
+ errorsDir.mkdir();
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ assertEquals(0, shell.getExitCode());
+ exec("createtable " + table, true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
+ @Test(timeout = 45000)
+ public void interpreter() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+ // Need to allow time for this to propagate through zoocache/zookeeper
+ UtilWaitThread.sleep(3000);
+
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void listcompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.minc.slow.opt.sleepTime=1000", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t " + table, true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void maxrow() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void merge() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f " + table);
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(2, output.get().split("\n").length);
+ exec("getsplits -t accumulo.root", true);
+ assertEquals(1, output.get().split("\n").length);
+ exec("merge --all -t " + MetadataTable.NAME + "");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(1, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void renametable() throws Exception {
+ final String table = name.getMethodName() + "1", rename = name.getMethodName() + "2";
+
+ exec("createtable " + table);
+ exec("insert this is a value");
+ exec("renametable " + table + " " + rename);
+ exec("tables", true, rename, true);
+ exec("tables", true, table, false);
+ exec("scan -t " + rename, true, "value", true);
+ exec("deletetable -f " + rename, true);
+ }
+
+ @Test(timeout = 30000)
+ public void tables() throws Exception {
+ final String table = name.getMethodName(), table1 = table + "_z", table2 = table + "_a";
+ exec("createtable " + table1);
+ exec("createtable " + table2);
+ exec("notable");
+ String lst = exec("tables -l");
+ assertTrue(lst.indexOf(table2) < lst.indexOf(table1));
+ lst = exec("tables -l -s");
+ assertTrue(lst.indexOf(table1) < lst.indexOf(table2));
+ }
+
+ @Test(timeout = 45000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(11, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
+ @Test(timeout = 45000)
+ public void listscans() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
++
++ // Should be about a 3 second scan
++ for (int i = 0; i < 6; i++) {
++ exec("insert " + i + " cf cq value", true);
++ }
+ exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
++ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Connector connector = getConnector();
+ Scanner s = connector.createScanner(table, Authorizations.EMPTY);
+ FunctionalTestUtils.count(s);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
- String hostPortPattern = ".+:\\d+";
- String tserver = parts[0].trim();
- assertTrue(tserver.matches(hostPortPattern));
- assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
- String client = parts[1].trim();
- assertTrue(client.matches(hostPortPattern));
- // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
++
++ List<String> scans = new ArrayList<String>();
++ // Try to find the active scan for about 5seconds
++ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
++ String currentScans = exec("listscans", true);
++ String[] lines = currentScans.split("\n");
++ for (int scanOffset = 2; i < lines.length; i++) {
++ String currentScan = lines[scanOffset];
++ if (currentScan.contains(table)) {
++ scans.add(currentScan);
++ }
++ }
++ UtilWaitThread.sleep(100);
++ }
+ thread.join();
++
++ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
++
++ for (String scan : scans) {
++ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
++ String parts[] = scan.split("\\|");
++ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
++ String tserver = parts[0].trim();
++ // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
++ String hostPortPattern = ".+:\\d+";
++ assertTrue(tserver.matches(hostPortPattern));
++ assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
++ String client = parts[1].trim();
++ assertTrue(client.matches(hostPortPattern));
++ }
++
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void testPertableClasspath() throws Exception {
+ final String table = name.getMethodName();
+
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar", getFolder());
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t " + table + " -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline -w " + table);
+ exec("online -w " + table);
+
+ exec("table " + table, true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f " + table, true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
+ @Test(timeout = 45000)
+ public void trace() throws Exception {
+ // Make sure to not collide with the "trace" table
+ final String table = name.getMethodName() + "Test";
+
+ exec("trace on", true);
+ exec("createtable " + table, true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f " + table);
+ exec("sleep 1");
+ String trace = exec("trace off");
+ System.out.println(trace);
+ assertTrue(trace.contains("sendMutations"));
+ assertTrue(trace.contains("startScan"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
+ @Test(timeout = 30000)
+ public void badLogin() throws Exception {
+ input.set(ROOT_PASSWORD + "\n");
+ String err = exec("user NoSuchUser", false);
+ assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser"));
+ }
+
+ @Test(timeout = 30000)
+ public void namespaces() throws Exception {
+ exec("namespaces", true, "\"\"", true); // default namespace, displayed as quoted empty string
+ exec("namespaces", true, Namespaces.ACCUMULO_NAMESPACE, true);
+ exec("createnamespace thing1", true);
+ String namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing1"));
+
+ exec("renamenamespace thing1 thing2");
+ namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing2"));
+ assertTrue(!namespaces.contains("thing1"));
+
+ // can't delete a namespace that still contains tables, unless you do -f
+ exec("createtable thing2.thingy", true);
+ exec("deletenamespace thing2");
+ exec("y");
+ exec("namespaces", true, "thing2", true);
+
+ exec("du -ns thing2", true, "thing2.thingy", true);
+
+ // all "TableOperation" commands can take a namespace
+ exec("offline -ns thing2", true);
+ exec("online -ns thing2", true);
+ exec("flush -ns thing2", true);
+ exec("compact -ns thing2", true);
+ exec("createnamespace testers3", true);
+ exec("createtable testers3.1", true);
+ exec("createtable testers3.2", true);
+ exec("deletetable -ns testers3 -f", true);
+ exec("tables", true, "testers3.1", false);
+ exec("namespaces", true, "testers3", true);
+ exec("deletenamespace testers3 -f", true);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -ns thing2 -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("listiter -ns thing2 -scan", true, "Summing", true);
+ exec("deleteiter -ns thing2 -n name -scan", true);
+ exec("createuser dude");
+ exec("pass");
+ exec("pass");
+ exec("grant Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+ exec("revoke Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+
+ // properties override and such
+ exec("config -ns thing2 -s table.file.max=44444", true);
+ exec("config -ns thing2", true, "44444", true);
+ exec("config -t thing2.thingy", true, "44444", true);
+ exec("config -t thing2.thingy -s table.file.max=55555", true);
+ exec("config -t thing2.thingy", true, "55555", true);
+
+ // can copy properties when creating
+ exec("createnamespace thing3 -cc thing2", true);
+ exec("config -ns thing3", true, "44444", true);
+
+ exec("deletenamespace -f thing2", true);
+ exec("namespaces", true, "thing2", false);
+ exec("tables", true, "thing2.thingy", false);
+
+ // put constraints on a namespace
+ exec("constraint -ns thing3 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ exec("createtable thing3.constrained", true);
+ exec("table thing3.constrained", true);
+ exec("constraint -d 1");
+ // should fail
+ exec("constraint -l", true, "NumericValueConstraint", true);
+ exec("insert r cf cq abc", false);
+ exec("constraint -ns thing3 -d 1");
+ exec("sleep 1");
+ exec("insert r cf cq abc", true);
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
+ @Test(timeout = 30000)
+ public void scans() throws Exception {
+ exec("createtable t");
+ make10();
+ String result = exec("scan -np -b row1 -e row1");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row3 -e row5");
+ assertEquals(4, result.split("\n").length);
+ result = exec("scan -np -r row3");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row:");
+ assertEquals(1, result.split("\n").length);
+ result = exec("scan -np -b row");
+ assertEquals(11, result.split("\n").length);
+ result = exec("scan -np -e row:");
+ assertEquals(11, result.split("\n").length);
+ exec("deletetable -f t");
+ }
+
+ @Test(timeout = 30000)
+ public void whoami() throws Exception {
+ assertTrue(exec("whoami", true).contains("root"));
+ input.set("secret\nsecret\n");
+ exec("createuser test_user");
+ exec("setauths -u test_user -s 12,3,4");
+ String auths = exec("getauths -u test_user");
+ assertTrue(auths.contains("3") && auths.contains("12") && auths.contains("4"));
+ input.set("secret\n");
+ exec("user test_user", true);
+ assertTrue(exec("whoami", true).contains("test_user"));
+ input.set(ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
++
++ private List<String> getFiles(String tableId) throws IOException {
++ output.clear();
+
- private int countFiles(String tableId) throws IOException {
+ exec("scan -t " + MetadataTable.NAME + " -np -c file -b " + tableId + " -e " + tableId + "~");
+
+ log.debug("countFiles(): " + output.get());
+
- return output.get().split("\n").length - 1;
++ String[] lines = StringUtils.split(output.get(), "\n");
++ output.clear();
++
++ if (0 == lines.length) {
++ return Collections.emptyList();
++ }
++
++ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
++ }
++
++ private int countFiles(String tableId) throws IOException {
++ return getFiles(tableId).size();
+ }
+
+ private String getTableId(String tableName) throws Exception {
+ Connector conn = getConnector();
+
+ for (int i = 0; i < 5; i++) {
+ Map<String,String> nameToId = conn.tableOperations().tableIdMap();
+ if (nameToId.containsKey(tableName)) {
+ return nameToId.get(tableName);
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+
+ fail("Could not find ID for table: " + tableName);
+ // Will never get here
+ return null;
+ }
+
+}
[06/16] git commit: ACCUMULO-1936 Make testInstanceOperations a more
reliable test.
Posted by el...@apache.org.
ACCUMULO-1936 Make testInstanceOperations a more reliable test.
Avoid pulling scans for users other than root and compactions for
compactions over tables other than the one we're looking at. Slow down
the slow iterator even more.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/72395e01
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/72395e01
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/72395e01
Branch: refs/heads/master
Commit: 72395e010d3f495507d03664057a63c3d690a206
Parents: e13c273
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:26:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/proxy/SimpleTest.java | 110 +++++++++++++------
1 file changed, 75 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/72395e01/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 5573d20..77e7b9e 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServer;
@@ -102,7 +103,8 @@ import org.junit.rules.TemporaryFolder;
* Call every method on the proxy and try to verify that it works.
*/
public class SimpleTest {
-
+ private static final Logger log = Logger.getLogger(SimpleTest.class);
+
public static TemporaryFolder folder = new TemporaryFolder();
public static final String TABLE_TEST = "test";
@@ -704,7 +706,7 @@ public class SimpleTest {
assertEquals(0, entries.results.size());
}
- @Test(timeout = 10000)
+ @Test(timeout = 60000)
public void testInstanceOperations() throws Exception {
int tservers = 0;
for (String tserver : client.getTabletServers(creds)) {
@@ -745,12 +747,14 @@ public class SimpleTest {
// create a table that's very slow, so we can look for scans/compactions
client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ // Should take 5 seconds to read every record
+ for (int i = 0; i < 20; i++) {
+ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
+ }
+
+ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
// scan
Thread t = new Thread() {
@@ -768,28 +772,45 @@ public class SimpleTest {
}
};
t.start();
+
// look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
+ List<ActiveScan> scans = new ArrayList<ActiveScan>();
+ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
+ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
+ for (ActiveScan scan : scansForServer) {
+ if ("root".equals(scan.getUser())) {
+ scans.add(scan);
+ }
+ }
+
if (!scans.isEmpty())
- break loop;
+ break;
UtilWaitThread.sleep(10);
}
}
t.join();
- assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
+ assertFalse(scans.isEmpty());
+ boolean found = false;
+ Map<String,String> map = null;
+ for (int i = 0; i < scans.size() && !found; i++) {
+ ActiveScan scan = scans.get(i);
+ if ("root".equals(scan.getUser())) {
+ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
+ assertEquals(ScanType.SINGLE, scan.getType());
+ assertEquals("slow", scan.getTable());
+
+ map = client.tableIdMap(creds);
+ assertEquals(map.get("slow"), scan.getExtent().tableId);
+ assertTrue(scan.getExtent().endRow == null);
+ assertTrue(scan.getExtent().prevEndRow == null);
+ found = true;
+ }
+ }
+
+ assertTrue("Could not find a scan against the 'slow' table", found);
+
// start a compaction
t = new Thread() {
@Override
@@ -803,27 +824,46 @@ public class SimpleTest {
}
};
t.start();
-
+
+ final String desiredTableId = map.get("slow");
+
// try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
+ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
+ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
+ // Iterate over the tservers
for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
+ // And get the compactions on each
+ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
+ for (ActiveCompaction compact : compactionsOnServer) {
+ // There might be other compactions occurring (e.g. on METADATA) in which
+ // case we want to prune out those that aren't for our slow table
+ if (desiredTableId.equals(compact.getExtent().tableId)) {
+ compactions.add(compact);
+ }
+ }
+
+ // If we found a compaction for the table we wanted, so we can stop looking
if (!compactions.isEmpty())
- break loop2;
+ break;
}
UtilWaitThread.sleep(10);
}
t.join();
+
// verify the compaction information
assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
+ for (ActiveCompaction c : compactions) {
+ if (desiredTableId.equals(c.getExtent().tableId)) {
+ assertTrue(c.inputFiles.isEmpty());
+ assertEquals(CompactionType.MINOR, c.getType());
+ assertEquals(CompactionReason.USER, c.getReason());
+ assertEquals("", c.localityGroup);
+ assertTrue(c.outputFile.contains("default_tablet"));
+
+ return;
+ }
+ }
+ fail("Expection to find running compaction for table 'slow' but did not find one");
}
@Test
@@ -1178,5 +1218,5 @@ public class SimpleTest {
accumulo.stop();
folder.delete();
}
-
-}
+
+}
\ No newline at end of file
[04/16] git commit: ACCUMULO-2447 Always set PrintWriter on Shell and
don't print to stderr.
Posted by el...@apache.org.
ACCUMULO-2447 Always set PrintWriter on Shell and don't print to stderr.
While we don't have access to the printwriter/outputstream from the jline
ConsoleReader, we can force the user to pass in one (or just make one for
stdout). This will allow unit tests to correctly squash all output into
a bytearray (or similar) instead of having to redirect stderr too (which
would have other repercussions).
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e13c2733
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e13c2733
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e13c2733
Branch: refs/heads/master
Commit: e13c27331a42d55408427f18d97b285372f92a47
Parents: bc9cee0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:18:23 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +++++++-------------
.../accumulo/core/util/shell/ShellTest.java | 4 +++-
.../apache/accumulo/test/ShellServerTest.java | 3 ++-
3 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 656f2ae..6943b37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -202,16 +202,14 @@ public class Shell extends ShellOptions {
private boolean masking = false;
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
@@ -943,13 +941,9 @@ public class Shell extends ShellOptions {
}
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
public int getExitCode() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 02e9543..8505370 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
import jline.ConsoleReader;
@@ -81,7 +82,8 @@ public class ShellTest {
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index 28a4dd2..e2ebbcf 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -163,7 +163,8 @@ public class ShellServerTest {
public void setupShell() throws Exception {
// start the shell
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("-u", "root", "-p", secret, "-z", cluster.getInstanceName(), cluster.getZooKeepers());
exec("quit", true);