You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/01/23 19:05:41 UTC
[01/15] git commit: ACCUMULO-2183 Look for the thrift
SecurityErrorCode since this is coming from the ThriftSecurityException.
Updated Branches:
refs/heads/1.5.1-SNAPSHOT fbed7afab -> a147acdd6
refs/heads/1.6.0-SNAPSHOT ecdd8528d -> 9440d15cb
refs/heads/master 9496199d0 -> 88a9b53d6
ACCUMULO-2183 Look for the thrift SecurityErrorCode since this is coming from the ThriftSecurityException.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/956c32f8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/956c32f8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/956c32f8
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 956c32f8fa258ea0b87f267675cdcec91e16dce1
Parents: fbed7af
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:15:10 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:15:10 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/test/randomwalk/security/Validate.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/956c32f8/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
index 2a3e445..047bba3 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
@@ -108,7 +108,7 @@ public class Validate extends Test {
auths = WalkingSecurity.get(state).getUserAuthorizations(WalkingSecurity.get(state).getTabCredentials());
accuAuths = conn.securityOperations().getUserAuthorizations(WalkingSecurity.get(state).getTabUserName());
} catch (ThriftSecurityException ae) {
- if (ae.getCode().equals(SecurityErrorCode.USER_DOESNT_EXIST)) {
+ if (ae.getCode().equals(org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.USER_DOESNT_EXIST)) {
if (tableUserExists)
throw new AccumuloException("Table user didn't exist when they should.", ae);
else
[04/15] git commit: ACCUMULO-2234 Fix up formatting for madrob.
Posted by el...@apache.org.
ACCUMULO-2234 Fix up formatting for madrob.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9cf94f93
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9cf94f93
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9cf94f93
Branch: refs/heads/master
Commit: 9cf94f93685faaccc927e148791d50570bfb2f30
Parents: de7d198
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:19:30 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:19:30 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/InputFormatBase.java | 2 +-
.../test/continuous/ContinuousVerify.java | 74 ++++++++++----------
2 files changed, 38 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf94f93/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 9c23246..705ddbd 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -737,7 +737,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
protected static Instance getInstance(Configuration conf) {
if (conf.getBoolean(MOCK, false))
return new MockInstance(conf.get(INSTANCE_NAME));
-
+
ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
// Wrap the DefaultConfiguration with a SiteConfiguration
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf94f93/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index 6546eea..4a5c0cf 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -55,22 +55,22 @@ import org.apache.log4j.Logger;
public class ContinuousVerify extends Configured implements Tool {
private static final Logger log = Logger.getLogger(ContinuousVerify.class);
-
+
public static final VLongWritable DEF = new VLongWritable(-1);
-
+
public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
-
+
private LongWritable row = new LongWritable();
private LongWritable ref = new LongWritable();
private VLongWritable vrow = new VLongWritable();
-
+
private long corrupt = 0;
-
+
public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
long r = Long.parseLong(key.getRow().toString(), 16);
if (r < 0)
throw new IllegalArgumentException();
-
+
try {
ContinuousWalk.validate(key, data);
} catch (BadChecksumException bce) {
@@ -83,12 +83,12 @@ public class ContinuousVerify extends Configured implements Tool {
corrupt++;
return;
}
-
+
row.set(r);
-
+
context.write(row, DEF);
byte[] val = data.get();
-
+
int offset = ContinuousWalk.getPrevRowOffset(val);
if (offset > 0) {
ref.set(Long.parseLong(new String(val, offset, 16), 16));
@@ -97,18 +97,18 @@ public class ContinuousVerify extends Configured implements Tool {
}
}
}
-
+
public static enum Counts {
UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
}
-
+
public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
private ArrayList<Long> refs = new ArrayList<Long>();
-
+
public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
-
+
int defCount = 0;
-
+
refs.clear();
for (VLongWritable type : values) {
if (type.get() == -1) {
@@ -117,7 +117,7 @@ public class ContinuousVerify extends Configured implements Tool {
refs.add(type.get());
}
}
-
+
if (defCount == 0 && refs.size() > 0) {
StringBuilder sb = new StringBuilder();
String comma = "";
@@ -126,25 +126,25 @@ public class ContinuousVerify extends Configured implements Tool {
comma = ",";
sb.append(new String(ContinuousIngest.genRow(ref)));
}
-
+
context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
context.getCounter(Counts.UNDEFINED).increment(1);
-
+
} else if (defCount > 0 && refs.size() == 0) {
context.getCounter(Counts.UNREFERENCED).increment(1);
} else {
context.getCounter(Counts.REFERENCED).increment(1);
}
-
+
}
}
-
+
@Override
public int run(String[] args) throws Exception {
-
+
String auths = "";
ArrayList<String> argsList = new ArrayList<String>();
-
+
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--auths")) {
auths = args[++i];
@@ -152,14 +152,14 @@ public class ContinuousVerify extends Configured implements Tool {
argsList.add(args[i]);
}
}
-
+
args = argsList.toArray(new String[0]);
if (args.length != 10) {
throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
+ " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
}
-
+
String instance = args[0];
String zookeepers = args[1];
String user = args[2];
@@ -170,10 +170,10 @@ public class ContinuousVerify extends Configured implements Tool {
String reducers = args[7];
boolean scanOffline = Boolean.parseBoolean(args[8]);
String siteFile = args[9];
-
+
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
-
+
String clone = table;
Connector conn = null;
if (scanOffline) {
@@ -205,43 +205,43 @@ public class ContinuousVerify extends Configured implements Tool {
} catch (Exception e) {
throw new IOException(e);
}
-
+
job.setMapperClass(CMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(VLongWritable.class);
-
+
job.setReducerClass(CReducer.class);
job.setNumReduceTasks(Integer.parseInt(reducers));
-
+
job.setOutputFormatClass(TextOutputFormat.class);
-
+
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
-
+
Path sitePath = new Path(siteFile);
Path siteParentPath = sitePath.getParent();
if (null == siteParentPath) {
siteParentPath = new Path("/");
}
-
+
URI siteUri = new URI("hdfs://" + siteFile);
-
+
log.info("Adding " + siteUri + " to DistributedCache");
-
+
// Make sure that accumulo-site.xml is available for mappers running offline scans
// as they need to correctly choose instance.dfs.dir for the installation
DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
TextOutputFormat.setOutputPath(job, new Path(outputdir));
-
+
job.waitForCompletion(true);
-
+
if (scanOffline) {
conn.tableOperations().delete(clone);
}
return job.isSuccessful() ? 0 : 1;
}
-
+
/**
*
* @param args
@@ -253,4 +253,4 @@ public class ContinuousVerify extends Configured implements Tool {
if (res != 0)
System.exit(res);
}
-}
\ No newline at end of file
+}
[09/15] git commit: ACCUMULO-2235 Fix up formatter for madrob and
make the du test a little less sensitive to other tests
Posted by el...@apache.org.
ACCUMULO-2235 Fix up formatter for madrob and make the du test a little less sensitive to other tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/00bd820e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/00bd820e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/00bd820e
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 00bd820ef9ff016305241504eb83e3c6e90dd873
Parents: 956c32f
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:20:46 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:20:46 2014 -0500
----------------------------------------------------------------------
.../core/util/shell/commands/DUCommand.java | 26 ++--
.../apache/accumulo/test/ShellServerTest.java | 156 ++++++++++---------
2 files changed, 93 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/00bd820e/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
index b03f516..ca80e37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
@@ -35,11 +35,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
public class DUCommand extends Command {
-
+
private Option optTablePattern, optHumanReadble;
-
+
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws IOException, TableNotFoundException {
-
+
final SortedSet<String> tablesToFlush = new TreeSet<String>(Arrays.asList(cl.getArgs()));
boolean prettyPrint = cl.hasOption(optHumanReadble.getOpt()) ? true : false;
@@ -52,12 +52,12 @@ public class DUCommand extends Command {
}
}
}
-
+
// If we didn't get any tables, and we have a table selected, add the current table
if (tablesToFlush.isEmpty() && !shellState.getTableName().isEmpty()) {
tablesToFlush.add(shellState.getTableName());
}
-
+
try {
final AccumuloConfiguration acuConf = new ConfigurationCopy(shellState.getConnector().instanceOperations().getSystemConfiguration());
TableDiskUsage.printDiskUsage(acuConf, tablesToFlush, FileSystem.get(new Configuration()), shellState.getConnector(), new Printer() {
@@ -69,40 +69,40 @@ public class DUCommand extends Command {
throw new RuntimeException(ex);
}
}
-
+
}, prettyPrint);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
return 0;
}
-
+
@Override
public String description() {
return "prints how much space, in bytes, is used by files referenced by a table. When multiple tables are specified it prints how much space, in bytes, is used by files shared between tables, if any.";
}
-
+
@Override
public Options getOptions() {
final Options o = new Options();
-
+
optTablePattern = new Option("p", "pattern", true, "regex pattern of table names");
optTablePattern.setArgName("pattern");
-
+
optHumanReadble = new Option("h", "human-readable", false, "format large sizes to human readable units");
optHumanReadble.setArgName("human readable output");
o.addOption(optTablePattern);
o.addOption(optHumanReadble);
-
+
return o;
}
-
+
@Override
public String usage() {
return getName() + " <table>{ <table>}";
}
-
+
@Override
public int numArgs() {
return Shell.NO_FIXED_ARG_LENGTH_CHECK;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/00bd820e/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index bc73c9f..80e56d9 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -64,34 +64,34 @@ import org.junit.rules.TemporaryFolder;
public class ShellServerTest {
public static class TestOutputStream extends OutputStream {
StringBuilder sb = new StringBuilder();
-
+
@Override
public void write(int b) throws IOException {
sb.append((char) (0xff & b));
}
-
+
public String get() {
return sb.toString();
}
-
+
public void clear() {
sb.setLength(0);
}
}
-
+
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
public static MiniAccumuloCluster cluster;
public static TestOutputStream output;
public static Shell shell;
private static Process traceProcess;
-
+
static String exec(String cmd) throws IOException {
output.clear();
shell.execCommand(cmd, true, true);
return output.get();
}
-
+
static String exec(String cmd, boolean expectGoodExit) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
@@ -100,11 +100,11 @@ public class ShellServerTest {
assertBadExit("", true);
return result;
}
-
+
static String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
return exec(cmd, expectGoodExit, expectString, true);
}
-
+
static String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
@@ -113,15 +113,15 @@ public class ShellServerTest {
assertBadExit(expectString, stringPresent);
return result;
}
-
+
static void assertGoodExit(String s, boolean stringPresent) {
Shell.log.debug(output.get());
assertEquals(0, shell.getExitCode());
-
+
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
-
+
static void assertBadExit(String s, boolean stringPresent) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
@@ -129,16 +129,16 @@ public class ShellServerTest {
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
shell.resetExitCode();
}
-
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
folder.create();
MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
cluster = new MiniAccumuloCluster(cfg);
cluster.start();
-
+
System.setProperty("HOME", folder.getRoot().getAbsolutePath());
-
+
// start the shell
output = new TestOutputStream();
shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
@@ -147,23 +147,23 @@ public class ShellServerTest {
exec("quit", true);
shell.start();
shell.setExit(false);
-
+
// use reflection to call this method so it does not need to be made public
Method method = cluster.getClass().getDeclaredMethod("exec", Class.class, String[].class);
method.setAccessible(true);
traceProcess = (Process) method.invoke(cluster, TraceServer.class, new String[0]);
-
+
// give the tracer some time to start
UtilWaitThread.sleep(1000);
}
-
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
cluster.stop();
traceProcess.destroy();
folder.delete();
}
-
+
@Test(timeout = 30000)
public void exporttableImporttable() throws Exception {
// exporttable / importtable
@@ -185,7 +185,7 @@ public class ShellServerTest {
exec("deletetable -f t", true);
exec("deletetable -f t2", true);
}
-
+
private DistCp newDistCp() {
try {
@SuppressWarnings("unchecked")
@@ -205,7 +205,7 @@ public class ShellServerTest {
}
throw new RuntimeException("Unexpected constructors for DistCp");
}
-
+
@Test(timeout = 30000)
public void setscaniterDeletescaniter() throws Exception {
// setscaniter, deletescaniter
@@ -219,9 +219,9 @@ public class ShellServerTest {
exec("deletescaniter -n name", true);
exec("scan", true, "1", true);
exec("deletetable -f t");
-
+
}
-
+
@Test(timeout = 30000)
public void execfile() throws Exception {
// execfile
@@ -230,9 +230,9 @@ public class ShellServerTest {
writer.println("about");
writer.close();
exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
-
+
}
-
+
@Test(timeout = 30000)
public void egrep() throws Exception {
// egrep
@@ -242,13 +242,17 @@ public class ShellServerTest {
assertTrue(lines.split("\n").length - 1 == 3);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void du() throws Exception {
+ // create and delete a table so we get out of a table context in the shell
+ exec("createtable du_test_table", true);
+ exec("deletetable -f du_test_table", true);
+
// Calling du not in a table context shouldn't throw an error
output.clear();
exec("du", true, "", true);
-
+
output.clear();
exec("createtable t");
make10();
@@ -258,10 +262,10 @@ public class ShellServerTest {
shell.execCommand("du -h", false, false);
String o = output.get();
// for some reason, there's a bit of fluctuation
- assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[t\\]\\n"));
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[t\\]\\n"));
exec("deletetable -f t");
}
-
+
@Test(timeout = 1000)
public void debug() throws Exception {
exec("debug", true, "off", true);
@@ -272,7 +276,7 @@ public class ShellServerTest {
exec("debug debug", false);
exec("debug debug debug", false);
}
-
+
@Test(timeout = 30000)
public void user() throws Exception {
// createuser, deleteuser, user, users, droptable, grant, revoke
@@ -306,7 +310,7 @@ public class ShellServerTest {
exec("deleteuser xyzzy", true);
exec("users", true, "xyzzy", false);
}
-
+
@Test(timeout = 30000)
public void iter() throws Exception {
// setshelliter, listshelliter, deleteshelliter
@@ -347,9 +351,9 @@ public class ShellServerTest {
exec("listiter -scan", true, "Iterator name", false);
exec("listiter -scan", true, "Iterator xyzzy", true);
exec("deletetable -f t");
-
+
}
-
+
@Test(timeout = 30000)
public void notable() throws Exception {
// notable
@@ -361,7 +365,7 @@ public class ShellServerTest {
assertFalse(output.get().contains(" xyzzy>"));
exec("deletetable -f xyzzy");
}
-
+
@Test(timeout = 30000)
public void sleep() throws Exception {
// sleep
@@ -371,7 +375,7 @@ public class ShellServerTest {
assertTrue(diff >= 200);
assertTrue(diff < 400);
}
-
+
@Test(timeout = 30000)
public void addauths() throws Exception {
// addauths
@@ -385,7 +389,7 @@ public class ShellServerTest {
exec("scan -s bar", true, "[foo]", false);
exec("deletetable -f xyzzy");
}
-
+
@Test(timeout = 30000)
public void byeQuitExit() throws Exception {
// bye, quit, exit
@@ -396,20 +400,20 @@ public class ShellServerTest {
shell.setExit(false);
}
}
-
+
@Test(timeout = 30000)
public void classpath() throws Exception {
// classpath
exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
}
-
+
@Test(timeout = 30000)
public void clearCls() throws Exception {
// clear/cls
exec("cls", true, "[1;1H");
exec("clear", true, "[2J");
}
-
+
@Test(timeout = 30000)
public void clonetable() throws Exception {
// clonetable
@@ -454,7 +458,7 @@ public class ShellServerTest {
exec("deletetable -f clone");
exec("deletetable -f c");
}
-
+
@Test(timeout = 30000)
public void constraint() throws Exception {
// constraint
@@ -467,7 +471,7 @@ public class ShellServerTest {
exec("constraint -l -t c", true, "VisibilityConstraint=1", false);
exec("deletetable -f c");
}
-
+
@Test(timeout = 30000)
public void deletemany() throws Exception {
// deletemany
@@ -494,7 +498,7 @@ public class ShellServerTest {
assertEquals(9, countkeys("t"));
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void deleterows() throws Exception {
// deleterows
@@ -508,7 +512,7 @@ public class ShellServerTest {
assertTrue(base + 2 == countFiles());
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void groups() throws Exception {
exec("createtable t");
@@ -517,7 +521,7 @@ public class ShellServerTest {
exec("getgroups -t t", true, "num=1,2,3", true);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void grep() throws Exception {
exec("createtable t", true);
@@ -526,7 +530,7 @@ public class ShellServerTest {
exec("grep row5", true, "row5", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void help() throws Exception {
exec("help -np", true, "Help Commands", true);
@@ -540,7 +544,7 @@ public class ShellServerTest {
exec("help " + c, true);
}
}
-
+
// @Test(timeout = 30000)
public void history() throws Exception {
exec("history -c", true);
@@ -549,7 +553,7 @@ public class ShellServerTest {
exec("history", true, "unusualstring", true);
exec("history", true, "history", true);
}
-
+
@Test(timeout = 30000)
public void importDirectory() throws Exception {
Configuration conf = new Configuration();
@@ -582,12 +586,12 @@ public class ShellServerTest {
exec("scan -r 00000099", true, "00000099", true);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void info() throws Exception {
exec("info", true, Constants.VERSION, true);
}
-
+
@Test(timeout = 30000)
public void interpreter() throws Exception {
exec("createtable t", true);
@@ -599,7 +603,7 @@ public class ShellServerTest {
exec("scan -b 02", true, "value", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void listcompactions() throws Exception {
exec("createtable t", true);
@@ -618,7 +622,7 @@ public class ShellServerTest {
assertEquals(12, parts.length);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void maxrow() throws Exception {
exec("createtable t", true);
@@ -631,7 +635,7 @@ public class ShellServerTest {
exec("maxrow", true, "ccc", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void merge() throws Exception {
exec("createtable t");
@@ -646,7 +650,7 @@ public class ShellServerTest {
exec("getsplits -t !METADATA", true);
assertEquals(2, output.get().split("\n").length);
}
-
+
@Test(timeout = 30000)
public void ping() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -655,11 +659,11 @@ public class ShellServerTest {
if (output.get().split("\n").length == 3)
break;
UtilWaitThread.sleep(1000);
-
+
}
assertEquals(3, output.get().split("\n").length);
}
-
+
@Test(timeout = 30000)
public void renametable() throws Exception {
exec("createtable aaaa");
@@ -670,7 +674,7 @@ public class ShellServerTest {
exec("scan -t xyzzy", true, "value", true);
exec("deletetable -f xyzzy", true);
}
-
+
@Test(timeout = 30000)
public void systempermission() throws Exception {
exec("systempermissions");
@@ -678,7 +682,7 @@ public class ShellServerTest {
exec("tablepermissions", true);
assertEquals(6, output.get().split("\n").length - 1);
}
-
+
@Test(timeout = 30000)
public void listscans() throws Exception {
exec("createtable t", true);
@@ -713,52 +717,52 @@ public class ShellServerTest {
thread.join();
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void testPertableClasspath() throws Exception {
File fooFilterJar = File.createTempFile("FooFilter", ".jar");
FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
fooFilterJar.deleteOnExit();
-
+
File fooConstraintJar = File.createTempFile("FooConstraint", ".jar");
FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
fooConstraintJar.deleteOnExit();
-
+
exec(
"config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
true);
-
+
exec("createtable ptc", true);
exec("config -t ptc -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
-
+
UtilWaitThread.sleep(200);
-
- // We can't use the setiter command as Filter implements OptionDescriber which
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
// forces us to enter more input that I don't know how to input
// Instead, we can just manually set the property on the table.
exec("config -t ptc -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
-
+
exec("insert foo f q v", true);
-
+
UtilWaitThread.sleep(100);
-
+
exec("scan -np", true, "foo", false);
-
+
exec("constraint -a FooConstraint", true);
-
+
exec("offline ptc");
UtilWaitThread.sleep(500);
exec("online ptc");
-
+
exec("table ptc", true);
exec("insert foo f q v", false);
exec("insert ok foo q v", true);
-
+
exec("deletetable -f ptc", true);
exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
-
+
}
-
+
@Test(timeout = 30000)
public void trace() throws Exception {
exec("trace on", true);
@@ -772,21 +776,21 @@ public class ShellServerTest {
assertTrue(trace.contains("update"));
assertTrue(trace.contains("DeleteTable"));
}
-
+
private int countkeys(String table) throws IOException {
exec("scan -np -t " + table);
return output.get().split("\n").length - 1;
}
-
+
private void make10() throws IOException {
for (int i = 0; i < 10; i++) {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
-
+
private int countFiles() throws IOException {
exec("scan -t !METADATA -np -c file");
return output.get().split("\n").length - 1;
}
-
+
}
[07/15] git commit: ACCUMULO-2235 Fix up formatter for madrob and
make the du test a little less sensitive to other tests
Posted by el...@apache.org.
ACCUMULO-2235 Fix up formatter for madrob and make the du test a little less sensitive to other tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/00bd820e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/00bd820e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/00bd820e
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 00bd820ef9ff016305241504eb83e3c6e90dd873
Parents: 956c32f
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:20:46 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:20:46 2014 -0500
----------------------------------------------------------------------
.../core/util/shell/commands/DUCommand.java | 26 ++--
.../apache/accumulo/test/ShellServerTest.java | 156 ++++++++++---------
2 files changed, 93 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/00bd820e/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
index b03f516..ca80e37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
@@ -35,11 +35,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
public class DUCommand extends Command {
-
+
private Option optTablePattern, optHumanReadble;
-
+
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws IOException, TableNotFoundException {
-
+
final SortedSet<String> tablesToFlush = new TreeSet<String>(Arrays.asList(cl.getArgs()));
boolean prettyPrint = cl.hasOption(optHumanReadble.getOpt()) ? true : false;
@@ -52,12 +52,12 @@ public class DUCommand extends Command {
}
}
}
-
+
// If we didn't get any tables, and we have a table selected, add the current table
if (tablesToFlush.isEmpty() && !shellState.getTableName().isEmpty()) {
tablesToFlush.add(shellState.getTableName());
}
-
+
try {
final AccumuloConfiguration acuConf = new ConfigurationCopy(shellState.getConnector().instanceOperations().getSystemConfiguration());
TableDiskUsage.printDiskUsage(acuConf, tablesToFlush, FileSystem.get(new Configuration()), shellState.getConnector(), new Printer() {
@@ -69,40 +69,40 @@ public class DUCommand extends Command {
throw new RuntimeException(ex);
}
}
-
+
}, prettyPrint);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
return 0;
}
-
+
@Override
public String description() {
return "prints how much space, in bytes, is used by files referenced by a table. When multiple tables are specified it prints how much space, in bytes, is used by files shared between tables, if any.";
}
-
+
@Override
public Options getOptions() {
final Options o = new Options();
-
+
optTablePattern = new Option("p", "pattern", true, "regex pattern of table names");
optTablePattern.setArgName("pattern");
-
+
optHumanReadble = new Option("h", "human-readable", false, "format large sizes to human readable units");
optHumanReadble.setArgName("human readable output");
o.addOption(optTablePattern);
o.addOption(optHumanReadble);
-
+
return o;
}
-
+
@Override
public String usage() {
return getName() + " <table>{ <table>}";
}
-
+
@Override
public int numArgs() {
return Shell.NO_FIXED_ARG_LENGTH_CHECK;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/00bd820e/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index bc73c9f..80e56d9 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -64,34 +64,34 @@ import org.junit.rules.TemporaryFolder;
public class ShellServerTest {
public static class TestOutputStream extends OutputStream {
StringBuilder sb = new StringBuilder();
-
+
@Override
public void write(int b) throws IOException {
sb.append((char) (0xff & b));
}
-
+
public String get() {
return sb.toString();
}
-
+
public void clear() {
sb.setLength(0);
}
}
-
+
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
public static MiniAccumuloCluster cluster;
public static TestOutputStream output;
public static Shell shell;
private static Process traceProcess;
-
+
static String exec(String cmd) throws IOException {
output.clear();
shell.execCommand(cmd, true, true);
return output.get();
}
-
+
static String exec(String cmd, boolean expectGoodExit) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
@@ -100,11 +100,11 @@ public class ShellServerTest {
assertBadExit("", true);
return result;
}
-
+
static String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
return exec(cmd, expectGoodExit, expectString, true);
}
-
+
static String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
@@ -113,15 +113,15 @@ public class ShellServerTest {
assertBadExit(expectString, stringPresent);
return result;
}
-
+
static void assertGoodExit(String s, boolean stringPresent) {
Shell.log.debug(output.get());
assertEquals(0, shell.getExitCode());
-
+
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
-
+
static void assertBadExit(String s, boolean stringPresent) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
@@ -129,16 +129,16 @@ public class ShellServerTest {
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
shell.resetExitCode();
}
-
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
folder.create();
MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
cluster = new MiniAccumuloCluster(cfg);
cluster.start();
-
+
System.setProperty("HOME", folder.getRoot().getAbsolutePath());
-
+
// start the shell
output = new TestOutputStream();
shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
@@ -147,23 +147,23 @@ public class ShellServerTest {
exec("quit", true);
shell.start();
shell.setExit(false);
-
+
// use reflection to call this method so it does not need to be made public
Method method = cluster.getClass().getDeclaredMethod("exec", Class.class, String[].class);
method.setAccessible(true);
traceProcess = (Process) method.invoke(cluster, TraceServer.class, new String[0]);
-
+
// give the tracer some time to start
UtilWaitThread.sleep(1000);
}
-
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
cluster.stop();
traceProcess.destroy();
folder.delete();
}
-
+
@Test(timeout = 30000)
public void exporttableImporttable() throws Exception {
// exporttable / importtable
@@ -185,7 +185,7 @@ public class ShellServerTest {
exec("deletetable -f t", true);
exec("deletetable -f t2", true);
}
-
+
private DistCp newDistCp() {
try {
@SuppressWarnings("unchecked")
@@ -205,7 +205,7 @@ public class ShellServerTest {
}
throw new RuntimeException("Unexpected constructors for DistCp");
}
-
+
@Test(timeout = 30000)
public void setscaniterDeletescaniter() throws Exception {
// setscaniter, deletescaniter
@@ -219,9 +219,9 @@ public class ShellServerTest {
exec("deletescaniter -n name", true);
exec("scan", true, "1", true);
exec("deletetable -f t");
-
+
}
-
+
@Test(timeout = 30000)
public void execfile() throws Exception {
// execfile
@@ -230,9 +230,9 @@ public class ShellServerTest {
writer.println("about");
writer.close();
exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
-
+
}
-
+
@Test(timeout = 30000)
public void egrep() throws Exception {
// egrep
@@ -242,13 +242,17 @@ public class ShellServerTest {
assertTrue(lines.split("\n").length - 1 == 3);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void du() throws Exception {
+ // create and delete a table so we get out of a table context in the shell
+ exec("createtable du_test_table", true);
+ exec("deletetable -f du_test_table", true);
+
// Calling du not in a table context shouldn't throw an error
output.clear();
exec("du", true, "", true);
-
+
output.clear();
exec("createtable t");
make10();
@@ -258,10 +262,10 @@ public class ShellServerTest {
shell.execCommand("du -h", false, false);
String o = output.get();
// for some reason, there's a bit of fluctuation
- assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[t\\]\\n"));
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[t\\]\\n"));
exec("deletetable -f t");
}
-
+
@Test(timeout = 1000)
public void debug() throws Exception {
exec("debug", true, "off", true);
@@ -272,7 +276,7 @@ public class ShellServerTest {
exec("debug debug", false);
exec("debug debug debug", false);
}
-
+
@Test(timeout = 30000)
public void user() throws Exception {
// createuser, deleteuser, user, users, droptable, grant, revoke
@@ -306,7 +310,7 @@ public class ShellServerTest {
exec("deleteuser xyzzy", true);
exec("users", true, "xyzzy", false);
}
-
+
@Test(timeout = 30000)
public void iter() throws Exception {
// setshelliter, listshelliter, deleteshelliter
@@ -347,9 +351,9 @@ public class ShellServerTest {
exec("listiter -scan", true, "Iterator name", false);
exec("listiter -scan", true, "Iterator xyzzy", true);
exec("deletetable -f t");
-
+
}
-
+
@Test(timeout = 30000)
public void notable() throws Exception {
// notable
@@ -361,7 +365,7 @@ public class ShellServerTest {
assertFalse(output.get().contains(" xyzzy>"));
exec("deletetable -f xyzzy");
}
-
+
@Test(timeout = 30000)
public void sleep() throws Exception {
// sleep
@@ -371,7 +375,7 @@ public class ShellServerTest {
assertTrue(diff >= 200);
assertTrue(diff < 400);
}
-
+
@Test(timeout = 30000)
public void addauths() throws Exception {
// addauths
@@ -385,7 +389,7 @@ public class ShellServerTest {
exec("scan -s bar", true, "[foo]", false);
exec("deletetable -f xyzzy");
}
-
+
@Test(timeout = 30000)
public void byeQuitExit() throws Exception {
// bye, quit, exit
@@ -396,20 +400,20 @@ public class ShellServerTest {
shell.setExit(false);
}
}
-
+
@Test(timeout = 30000)
public void classpath() throws Exception {
// classpath
exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
}
-
+
@Test(timeout = 30000)
public void clearCls() throws Exception {
// clear/cls
exec("cls", true, "[1;1H");
exec("clear", true, "[2J");
}
-
+
@Test(timeout = 30000)
public void clonetable() throws Exception {
// clonetable
@@ -454,7 +458,7 @@ public class ShellServerTest {
exec("deletetable -f clone");
exec("deletetable -f c");
}
-
+
@Test(timeout = 30000)
public void constraint() throws Exception {
// constraint
@@ -467,7 +471,7 @@ public class ShellServerTest {
exec("constraint -l -t c", true, "VisibilityConstraint=1", false);
exec("deletetable -f c");
}
-
+
@Test(timeout = 30000)
public void deletemany() throws Exception {
// deletemany
@@ -494,7 +498,7 @@ public class ShellServerTest {
assertEquals(9, countkeys("t"));
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void deleterows() throws Exception {
// deleterows
@@ -508,7 +512,7 @@ public class ShellServerTest {
assertTrue(base + 2 == countFiles());
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void groups() throws Exception {
exec("createtable t");
@@ -517,7 +521,7 @@ public class ShellServerTest {
exec("getgroups -t t", true, "num=1,2,3", true);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void grep() throws Exception {
exec("createtable t", true);
@@ -526,7 +530,7 @@ public class ShellServerTest {
exec("grep row5", true, "row5", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void help() throws Exception {
exec("help -np", true, "Help Commands", true);
@@ -540,7 +544,7 @@ public class ShellServerTest {
exec("help " + c, true);
}
}
-
+
// @Test(timeout = 30000)
public void history() throws Exception {
exec("history -c", true);
@@ -549,7 +553,7 @@ public class ShellServerTest {
exec("history", true, "unusualstring", true);
exec("history", true, "history", true);
}
-
+
@Test(timeout = 30000)
public void importDirectory() throws Exception {
Configuration conf = new Configuration();
@@ -582,12 +586,12 @@ public class ShellServerTest {
exec("scan -r 00000099", true, "00000099", true);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void info() throws Exception {
exec("info", true, Constants.VERSION, true);
}
-
+
@Test(timeout = 30000)
public void interpreter() throws Exception {
exec("createtable t", true);
@@ -599,7 +603,7 @@ public class ShellServerTest {
exec("scan -b 02", true, "value", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void listcompactions() throws Exception {
exec("createtable t", true);
@@ -618,7 +622,7 @@ public class ShellServerTest {
assertEquals(12, parts.length);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void maxrow() throws Exception {
exec("createtable t", true);
@@ -631,7 +635,7 @@ public class ShellServerTest {
exec("maxrow", true, "ccc", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void merge() throws Exception {
exec("createtable t");
@@ -646,7 +650,7 @@ public class ShellServerTest {
exec("getsplits -t !METADATA", true);
assertEquals(2, output.get().split("\n").length);
}
-
+
@Test(timeout = 30000)
public void ping() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -655,11 +659,11 @@ public class ShellServerTest {
if (output.get().split("\n").length == 3)
break;
UtilWaitThread.sleep(1000);
-
+
}
assertEquals(3, output.get().split("\n").length);
}
-
+
@Test(timeout = 30000)
public void renametable() throws Exception {
exec("createtable aaaa");
@@ -670,7 +674,7 @@ public class ShellServerTest {
exec("scan -t xyzzy", true, "value", true);
exec("deletetable -f xyzzy", true);
}
-
+
@Test(timeout = 30000)
public void systempermission() throws Exception {
exec("systempermissions");
@@ -678,7 +682,7 @@ public class ShellServerTest {
exec("tablepermissions", true);
assertEquals(6, output.get().split("\n").length - 1);
}
-
+
@Test(timeout = 30000)
public void listscans() throws Exception {
exec("createtable t", true);
@@ -713,52 +717,52 @@ public class ShellServerTest {
thread.join();
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void testPertableClasspath() throws Exception {
File fooFilterJar = File.createTempFile("FooFilter", ".jar");
FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
fooFilterJar.deleteOnExit();
-
+
File fooConstraintJar = File.createTempFile("FooConstraint", ".jar");
FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
fooConstraintJar.deleteOnExit();
-
+
exec(
"config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
true);
-
+
exec("createtable ptc", true);
exec("config -t ptc -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
-
+
UtilWaitThread.sleep(200);
-
- // We can't use the setiter command as Filter implements OptionDescriber which
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
// forces us to enter more input that I don't know how to input
// Instead, we can just manually set the property on the table.
exec("config -t ptc -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
-
+
exec("insert foo f q v", true);
-
+
UtilWaitThread.sleep(100);
-
+
exec("scan -np", true, "foo", false);
-
+
exec("constraint -a FooConstraint", true);
-
+
exec("offline ptc");
UtilWaitThread.sleep(500);
exec("online ptc");
-
+
exec("table ptc", true);
exec("insert foo f q v", false);
exec("insert ok foo q v", true);
-
+
exec("deletetable -f ptc", true);
exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
-
+
}
-
+
@Test(timeout = 30000)
public void trace() throws Exception {
exec("trace on", true);
@@ -772,21 +776,21 @@ public class ShellServerTest {
assertTrue(trace.contains("update"));
assertTrue(trace.contains("DeleteTable"));
}
-
+
private int countkeys(String table) throws IOException {
exec("scan -np -t " + table);
return output.get().split("\n").length - 1;
}
-
+
private void make10() throws IOException {
for (int i = 0; i < 10; i++) {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
-
+
private int countFiles() throws IOException {
exec("scan -t !METADATA -np -c file");
return output.get().split("\n").length - 1;
}
-
+
}
[06/15] git commit: ACCUMULO-2234 Fix up formatting for madrob.
Posted by el...@apache.org.
ACCUMULO-2234 Fix up formatting for madrob.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9cf94f93
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9cf94f93
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9cf94f93
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 9cf94f93685faaccc927e148791d50570bfb2f30
Parents: de7d198
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:19:30 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:19:30 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/InputFormatBase.java | 2 +-
.../test/continuous/ContinuousVerify.java | 74 ++++++++++----------
2 files changed, 38 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf94f93/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 9c23246..705ddbd 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -737,7 +737,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
protected static Instance getInstance(Configuration conf) {
if (conf.getBoolean(MOCK, false))
return new MockInstance(conf.get(INSTANCE_NAME));
-
+
ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
// Wrap the DefaultConfiguration with a SiteConfiguration
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf94f93/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index 6546eea..4a5c0cf 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -55,22 +55,22 @@ import org.apache.log4j.Logger;
public class ContinuousVerify extends Configured implements Tool {
private static final Logger log = Logger.getLogger(ContinuousVerify.class);
-
+
public static final VLongWritable DEF = new VLongWritable(-1);
-
+
public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
-
+
private LongWritable row = new LongWritable();
private LongWritable ref = new LongWritable();
private VLongWritable vrow = new VLongWritable();
-
+
private long corrupt = 0;
-
+
public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
long r = Long.parseLong(key.getRow().toString(), 16);
if (r < 0)
throw new IllegalArgumentException();
-
+
try {
ContinuousWalk.validate(key, data);
} catch (BadChecksumException bce) {
@@ -83,12 +83,12 @@ public class ContinuousVerify extends Configured implements Tool {
corrupt++;
return;
}
-
+
row.set(r);
-
+
context.write(row, DEF);
byte[] val = data.get();
-
+
int offset = ContinuousWalk.getPrevRowOffset(val);
if (offset > 0) {
ref.set(Long.parseLong(new String(val, offset, 16), 16));
@@ -97,18 +97,18 @@ public class ContinuousVerify extends Configured implements Tool {
}
}
}
-
+
public static enum Counts {
UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
}
-
+
public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
private ArrayList<Long> refs = new ArrayList<Long>();
-
+
public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
-
+
int defCount = 0;
-
+
refs.clear();
for (VLongWritable type : values) {
if (type.get() == -1) {
@@ -117,7 +117,7 @@ public class ContinuousVerify extends Configured implements Tool {
refs.add(type.get());
}
}
-
+
if (defCount == 0 && refs.size() > 0) {
StringBuilder sb = new StringBuilder();
String comma = "";
@@ -126,25 +126,25 @@ public class ContinuousVerify extends Configured implements Tool {
comma = ",";
sb.append(new String(ContinuousIngest.genRow(ref)));
}
-
+
context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
context.getCounter(Counts.UNDEFINED).increment(1);
-
+
} else if (defCount > 0 && refs.size() == 0) {
context.getCounter(Counts.UNREFERENCED).increment(1);
} else {
context.getCounter(Counts.REFERENCED).increment(1);
}
-
+
}
}
-
+
@Override
public int run(String[] args) throws Exception {
-
+
String auths = "";
ArrayList<String> argsList = new ArrayList<String>();
-
+
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--auths")) {
auths = args[++i];
@@ -152,14 +152,14 @@ public class ContinuousVerify extends Configured implements Tool {
argsList.add(args[i]);
}
}
-
+
args = argsList.toArray(new String[0]);
if (args.length != 10) {
throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
+ " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
}
-
+
String instance = args[0];
String zookeepers = args[1];
String user = args[2];
@@ -170,10 +170,10 @@ public class ContinuousVerify extends Configured implements Tool {
String reducers = args[7];
boolean scanOffline = Boolean.parseBoolean(args[8]);
String siteFile = args[9];
-
+
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
-
+
String clone = table;
Connector conn = null;
if (scanOffline) {
@@ -205,43 +205,43 @@ public class ContinuousVerify extends Configured implements Tool {
} catch (Exception e) {
throw new IOException(e);
}
-
+
job.setMapperClass(CMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(VLongWritable.class);
-
+
job.setReducerClass(CReducer.class);
job.setNumReduceTasks(Integer.parseInt(reducers));
-
+
job.setOutputFormatClass(TextOutputFormat.class);
-
+
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
-
+
Path sitePath = new Path(siteFile);
Path siteParentPath = sitePath.getParent();
if (null == siteParentPath) {
siteParentPath = new Path("/");
}
-
+
URI siteUri = new URI("hdfs://" + siteFile);
-
+
log.info("Adding " + siteUri + " to DistributedCache");
-
+
// Make sure that accumulo-site.xml is available for mappers running offline scans
// as they need to correctly choose instance.dfs.dir for the installation
DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
TextOutputFormat.setOutputPath(job, new Path(outputdir));
-
+
job.waitForCompletion(true);
-
+
if (scanOffline) {
conn.tableOperations().delete(clone);
}
return job.isSuccessful() ? 0 : 1;
}
-
+
/**
*
* @param args
@@ -253,4 +253,4 @@ public class ContinuousVerify extends Configured implements Tool {
if (res != 0)
System.exit(res);
}
-}
\ No newline at end of file
+}
[10/15] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a147acdd
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a147acdd
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a147acdd
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: a147acdd600f2fce2a78c9845d00ac3779185707
Parents: 00bd820 9cf94f9
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:31:48 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:31:48 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ++++++++++++++++++
.../test/continuous/ContinuousVerify.java | 82 +-
3 files changed, 1676 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a147acdd/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 592cde6,0000000..f7b2263
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -1,442 -1,0 +1,442 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.log4j.Level;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
+ */
+public class RangeInputSplit extends InputSplit implements Writable {
+ private Range range;
+ private String[] locations;
+ private String table, instanceName, zooKeepers, principal;
+ private AuthenticationToken token;
+ private Boolean offline, mockInstance, isolatedScan, localIterators;
+ private Authorizations auths;
+ private Set<Pair<Text,Text>> fetchedColumns;
+ private List<IteratorSetting> iterators;
+ private Level level;
+
+ public RangeInputSplit() {
+ range = new Range();
+ locations = new String[0];
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ this.range = range;
+ this.locations = locations;
+ }
+
+ public Range getRange() {
+ return range;
+ }
+
+ private static byte[] extractBytes(ByteSequence seq, int numBytes) {
+ byte[] bytes = new byte[numBytes + 1];
+ bytes[0] = 0;
+ for (int i = 0; i < numBytes; i++) {
+ if (i >= seq.length())
+ bytes[i + 1] = 0;
+ else
+ bytes[i + 1] = seq.byteAt(i);
+ }
+ return bytes;
+ }
+
+ public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
+ int maxDepth = Math.min(Math.max(end.length(), start.length()), position.length());
+ BigInteger startBI = new BigInteger(extractBytes(start, maxDepth));
+ BigInteger endBI = new BigInteger(extractBytes(end, maxDepth));
+ BigInteger positionBI = new BigInteger(extractBytes(position, maxDepth));
+ return (float) (positionBI.subtract(startBI).doubleValue() / endBI.subtract(startBI).doubleValue());
+ }
+
+ public float getProgress(Key currentKey) {
+ if (currentKey == null)
+ return 0f;
+ if (range.getStartKey() != null && range.getEndKey() != null) {
+ if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
+ // just look at the row progress
+ return getProgress(range.getStartKey().getRowData(), range.getEndKey().getRowData(), currentKey.getRowData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM) != 0) {
+ // just look at the column family progress
+ return getProgress(range.getStartKey().getColumnFamilyData(), range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM_COLQUAL) != 0) {
+ // just look at the column qualifier progress
+ return getProgress(range.getStartKey().getColumnQualifierData(), range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
+ }
+ }
+ // if we can't figure it out, then claim no progress
+ return 0f;
+ }
+
+ /**
+ * This implementation of length is only an estimate, it does not provide exact values. Do not have your code rely on this return value.
+ */
+ @Override
+ public long getLength() throws IOException {
+ Text startRow = range.isInfiniteStartKey() ? new Text(new byte[] {Byte.MIN_VALUE}) : range.getStartKey().getRow();
+ Text stopRow = range.isInfiniteStopKey() ? new Text(new byte[] {Byte.MAX_VALUE}) : range.getEndKey().getRow();
+ int maxCommon = Math.min(7, Math.min(startRow.getLength(), stopRow.getLength()));
+ long diff = 0;
+
+ byte[] start = startRow.getBytes();
+ byte[] stop = stopRow.getBytes();
+ for (int i = 0; i < maxCommon; ++i) {
+ diff |= 0xff & (start[i] ^ stop[i]);
+ diff <<= Byte.SIZE;
+ }
+
+ if (startRow.getLength() != stopRow.getLength())
+ diff |= 0xff;
+
+ return diff + 1;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException {
+ return locations;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ range.readFields(in);
+ int numLocs = in.readInt();
+ locations = new String[numLocs];
+ for (int i = 0; i < numLocs; ++i)
+ locations[i] = in.readUTF();
+
+ if (in.readBoolean()) {
+ isolatedScan = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ offline = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ localIterators = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ mockInstance = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ int numColumns = in.readInt();
+ List<String> columns = new ArrayList<String>(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+ columns.add(in.readUTF());
+ }
+
+ fetchedColumns = InputConfigurator.deserializeFetchedColumns(columns);
+ }
+
+ if (in.readBoolean()) {
+ String strAuths = in.readUTF();
+ auths = new Authorizations(strAuths.getBytes(Charset.forName("UTF-8")));
+ }
+
+ if (in.readBoolean()) {
+ principal = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ String tokenClass = in.readUTF();
+ byte[] base64TokenBytes = in.readUTF().getBytes(Charset.forName("UTF-8"));
+ byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
+
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ if (in.readBoolean()) {
+ instanceName = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ zooKeepers = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ level = Level.toLevel(in.readInt());
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ range.write(out);
+ out.writeInt(locations.length);
+ for (int i = 0; i < locations.length; ++i)
+ out.writeUTF(locations[i]);
+
+ out.writeBoolean(null != isolatedScan);
+ if (null != isolatedScan) {
+ out.writeBoolean(isolatedScan);
+ }
+
+ out.writeBoolean(null != offline);
+ if (null != offline) {
+ out.writeBoolean(offline);
+ }
+
+ out.writeBoolean(null != localIterators);
+ if (null != localIterators) {
+ out.writeBoolean(localIterators);
+ }
+
+ out.writeBoolean(null != mockInstance);
+ if (null != mockInstance) {
+ out.writeBoolean(mockInstance);
+ }
+
+ out.writeBoolean(null != fetchedColumns);
+ if (null != fetchedColumns) {
+ String[] cols = InputConfigurator.serializeColumns(fetchedColumns);
+ out.writeInt(cols.length);
+ for (String col : cols) {
+ out.writeUTF(col);
+ }
+ }
+
+ out.writeBoolean(null != auths);
+ if (null != auths) {
+ out.writeUTF(auths.serialize());
+ }
+
+ out.writeBoolean(null != principal);
+ if (null != principal) {
+ out.writeUTF(principal);
+ }
+
+ out.writeBoolean(null != token);
+ if (null != token) {
+ out.writeUTF(token.getClass().getCanonicalName());
+ try {
+ out.writeUTF(CredentialHelper.tokenAsBase64(token));
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ out.writeBoolean(null != instanceName);
+ if (null != instanceName) {
+ out.writeUTF(instanceName);
+ }
+
+ out.writeBoolean(null != zooKeepers);
+ if (null != zooKeepers) {
+ out.writeUTF(zooKeepers);
+ }
+
+ out.writeBoolean(null != level);
+ if (null != level) {
+ out.writeInt(level.toInt());
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ sb.append("Range: ").append(range);
+ sb.append(" Locations: ").append(Arrays.asList(locations));
+ sb.append(" Table: ").append(table);
+ sb.append(" InstanceName: ").append(instanceName);
+ sb.append(" zooKeepers: ").append(zooKeepers);
+ sb.append(" principal: ").append(principal);
+ sb.append(" authenticationToken: ").append(token);
+ sb.append(" Authorizations: ").append(auths);
+ sb.append(" offlineScan: ").append(offline);
+ sb.append(" mockInstance: ").append(mockInstance);
+ sb.append(" isolatedScan: ").append(isolatedScan);
+ sb.append(" localIterators: ").append(localIterators);
+ sb.append(" fetchColumns: ").append(fetchedColumns);
+ sb.append(" iterators: ").append(iterators);
+ sb.append(" logLevel: ").append(level);
+ return sb.toString();
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public Instance getInstance() {
+ if (null == instanceName) {
+ return null;
+ }
+
+ if (isMockInstance()) {
+ return new MockInstance(getInstanceName());
+ }
+
+ if (null == zooKeepers) {
+ return null;
+ }
-
++
+ ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+
+ // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
+ // when it's present
+ AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+ zki.setConfiguration(xmlConfig);
+
+ return zki;
+ }
+
+ public String getInstanceName() {
+ return instanceName;
+ }
+
+ public void setInstanceName(String instanceName) {
+ this.instanceName = instanceName;
+ }
+
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ public void setZooKeepers(String zooKeepers) {
+ this.zooKeepers = zooKeepers;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ public void setToken(AuthenticationToken token) {
+ this.token = token;
+ ;
+ }
+
+ public Boolean isOffline() {
+ return offline;
+ }
+
+ public void setOffline(Boolean offline) {
+ this.offline = offline;
+ }
+
+ public void setLocations(String[] locations) {
+ this.locations = locations;
+ }
+
+ public Boolean isMockInstance() {
+ return mockInstance;
+ }
+
+ public void setMockInstance(Boolean mockInstance) {
+ this.mockInstance = mockInstance;
+ }
+
+ public Boolean isIsolatedScan() {
+ return isolatedScan;
+ }
+
+ public void setIsolatedScan(Boolean isolatedScan) {
+ this.isolatedScan = isolatedScan;
+ }
+
+ public Authorizations getAuths() {
+ return auths;
+ }
+
+ public void setAuths(Authorizations auths) {
+ this.auths = auths;
+ }
+
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ public Boolean usesLocalIterators() {
+ return localIterators;
+ }
+
+ public void setUsesLocalIterators(Boolean localIterators) {
+ this.localIterators = localIterators;
+ }
+
+ public Set<Pair<Text,Text>> getFetchedColumns() {
+ return fetchedColumns;
+ }
+
+ public void setFetchedColumns(Set<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = fetchedColumns;
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+
+ public void setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ }
+
+ public Level getLogLevel() {
+ return level;
+ }
+
+ public void setLogLevel(Level level) {
+ this.level = level;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a147acdd/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 69a483f,0000000..a9d7153
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -1,265 -1,0 +1,265 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.test.continuous.ContinuousWalk.BadChecksumException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
+ */
+
+public class ContinuousVerify extends Configured implements Tool {
+ private static final Logger log = Logger.getLogger(ContinuousVerify.class);
-
++
+ // work around hadoop-1/hadoop-2 runtime incompatibility
+ static private Method INCREMENT;
+ static {
+ try {
+ INCREMENT = Counter.class.getMethod("increment", Long.TYPE);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
-
++
+ static void increment(Object obj) {
+ try {
+ INCREMENT.invoke(obj, 1L);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
-
++
+ public static final VLongWritable DEF = new VLongWritable(-1);
-
++
+ public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
-
++
+ private LongWritable row = new LongWritable();
+ private LongWritable ref = new LongWritable();
+ private VLongWritable vrow = new VLongWritable();
+
+ private long corrupt = 0;
-
++
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+ long r = Long.parseLong(key.getRow().toString(), 16);
+ if (r < 0)
+ throw new IllegalArgumentException();
-
++
+ try {
+ ContinuousWalk.validate(key, data);
+ } catch (BadChecksumException bce) {
+ increment(context.getCounter(Counts.CORRUPT));
+ if (corrupt < 1000) {
+ System.out.println("ERROR Bad checksum : " + key);
+ } else if (corrupt == 1000) {
+ System.out.println("Too many bad checksums, not printing anymore!");
+ }
+ corrupt++;
+ return;
+ }
-
++
+ row.set(r);
-
++
+ context.write(row, DEF);
+ byte[] val = data.get();
-
++
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ ref.set(Long.parseLong(new String(val, offset, 16), 16));
+ vrow.set(r);
+ context.write(ref, vrow);
+ }
+ }
+ }
-
++
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ }
-
++
+ public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+ private ArrayList<Long> refs = new ArrayList<Long>();
-
++
+ @Override
+ public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
-
++
+ int defCount = 0;
-
++
+ refs.clear();
+ for (VLongWritable type : values) {
+ if (type.get() == -1) {
+ defCount++;
+ } else {
+ refs.add(type.get());
+ }
+ }
-
++
+ if (defCount == 0 && refs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ String comma = "";
+ for (Long ref : refs) {
+ sb.append(comma);
+ comma = ",";
+ sb.append(new String(ContinuousIngest.genRow(ref)));
+ }
-
++
+ context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
+ increment(context.getCounter(Counts.UNDEFINED));
-
++
+ } else if (defCount > 0 && refs.size() == 0) {
+ increment(context.getCounter(Counts.UNREFERENCED));
+ } else {
+ increment(context.getCounter(Counts.REFERENCED));
+ }
-
++
+ }
+ }
-
++
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names = "--output", description = "location in HDFS to store the results; must not exist", required = true)
+ String outputDir = "/tmp/continuousVerify";
-
++
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
-
++
+ @Parameter(names = "--reducers", description = "the number of reducers to use", required = true, validateWith = PositiveInteger.class)
+ int reducers = 0;
-
++
+ @Parameter(names = "--offline", description = "perform the verification directly on the files while the table is offline")
+ boolean scanOffline = false;
-
++
+ @Parameter(names = "--sitefile", description = "location of accumulo-site.xml in HDFS", required = true)
+ String siteFile;
-
++
+ public Opts() {
+ super("ci");
+ }
+ }
-
++
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(this.getClass().getName(), args);
-
++
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
-
++
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ String clone = opts.getTableName();
+ Connector conn = null;
+ if (opts.scanOffline) {
+ Random random = new Random();
+ clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
+ conn = opts.getConnector();
+ conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
+ conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
+ }
-
++
+ // set up ranges
+ try {
+ Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
-
++
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
-
++
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(opts.reducers);
-
++
+ job.setOutputFormatClass(TextOutputFormat.class);
-
++
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);
-
++
+ TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));
-
++
+ Path sitePath = new Path(opts.siteFile);
+ Path siteParentPath = sitePath.getParent();
+ if (null == siteParentPath) {
+ siteParentPath = new Path("/");
+ }
-
++
+ URI siteUri = new URI("hdfs://" + opts.siteFile);
-
++
+ log.info("Adding " + siteUri + " to DistributedCache");
-
++
+ // Make sure that accumulo-site.xml is available for mappers running offline scans
+ // as they need to correctly choose instance.dfs.dir for the installation
+ DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
-
++
+ job.waitForCompletion(true);
-
++
+ if (opts.scanOffline) {
+ conn.tableOperations().delete(clone);
+ }
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
-
++
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
[15/15] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/88a9b53d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/88a9b53d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/88a9b53d
Branch: refs/heads/master
Commit: 88a9b53d645e580fcae00936e73f18a6c0729ddf
Parents: 9496199 9440d15
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 13:05:22 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 13:05:22 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../core/util/shell/commands/DUCommand.java | 4 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ++++++++++++++++++
.../test/continuous/ContinuousVerify.java | 80 +-
.../org/apache/accumulo/test/ShellServerIT.java | 112 +-
5 files changed, 1734 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
[03/15] git commit: ACCUMULO-2183 Look for the thrift
SecurityErrorCode since this is coming from the ThriftSecurityException.
Posted by el...@apache.org.
ACCUMULO-2183 Look for the thrift SecurityErrorCode since this is coming from the ThriftSecurityException.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/956c32f8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/956c32f8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/956c32f8
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 956c32f8fa258ea0b87f267675cdcec91e16dce1
Parents: fbed7af
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:15:10 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:15:10 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/test/randomwalk/security/Validate.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/956c32f8/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
index 2a3e445..047bba3 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
@@ -108,7 +108,7 @@ public class Validate extends Test {
auths = WalkingSecurity.get(state).getUserAuthorizations(WalkingSecurity.get(state).getTabCredentials());
accuAuths = conn.securityOperations().getUserAuthorizations(WalkingSecurity.get(state).getTabUserName());
} catch (ThriftSecurityException ae) {
- if (ae.getCode().equals(SecurityErrorCode.USER_DOESNT_EXIST)) {
+ if (ae.getCode().equals(org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.USER_DOESNT_EXIST)) {
if (tableUserExists)
throw new AccumuloException("Table user didn't exist when they should.", ae);
else
[11/15] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a147acdd
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a147acdd
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a147acdd
Branch: refs/heads/master
Commit: a147acdd600f2fce2a78c9845d00ac3779185707
Parents: 00bd820 9cf94f9
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:31:48 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:31:48 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ++++++++++++++++++
.../test/continuous/ContinuousVerify.java | 82 +-
3 files changed, 1676 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a147acdd/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 592cde6,0000000..f7b2263
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -1,442 -1,0 +1,442 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.log4j.Level;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
+ */
+public class RangeInputSplit extends InputSplit implements Writable {
+ private Range range;
+ private String[] locations;
+ private String table, instanceName, zooKeepers, principal;
+ private AuthenticationToken token;
+ private Boolean offline, mockInstance, isolatedScan, localIterators;
+ private Authorizations auths;
+ private Set<Pair<Text,Text>> fetchedColumns;
+ private List<IteratorSetting> iterators;
+ private Level level;
+
+ public RangeInputSplit() {
+ range = new Range();
+ locations = new String[0];
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ this.range = range;
+ this.locations = locations;
+ }
+
+ public Range getRange() {
+ return range;
+ }
+
+ private static byte[] extractBytes(ByteSequence seq, int numBytes) {
+ byte[] bytes = new byte[numBytes + 1];
+ bytes[0] = 0;
+ for (int i = 0; i < numBytes; i++) {
+ if (i >= seq.length())
+ bytes[i + 1] = 0;
+ else
+ bytes[i + 1] = seq.byteAt(i);
+ }
+ return bytes;
+ }
+
+ public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
+ int maxDepth = Math.min(Math.max(end.length(), start.length()), position.length());
+ BigInteger startBI = new BigInteger(extractBytes(start, maxDepth));
+ BigInteger endBI = new BigInteger(extractBytes(end, maxDepth));
+ BigInteger positionBI = new BigInteger(extractBytes(position, maxDepth));
+ return (float) (positionBI.subtract(startBI).doubleValue() / endBI.subtract(startBI).doubleValue());
+ }
+
+ public float getProgress(Key currentKey) {
+ if (currentKey == null)
+ return 0f;
+ if (range.getStartKey() != null && range.getEndKey() != null) {
+ if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
+ // just look at the row progress
+ return getProgress(range.getStartKey().getRowData(), range.getEndKey().getRowData(), currentKey.getRowData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM) != 0) {
+ // just look at the column family progress
+ return getProgress(range.getStartKey().getColumnFamilyData(), range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM_COLQUAL) != 0) {
+ // just look at the column qualifier progress
+ return getProgress(range.getStartKey().getColumnQualifierData(), range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
+ }
+ }
+ // if we can't figure it out, then claim no progress
+ return 0f;
+ }
+
+ /**
+ * This implementation of length is only an estimate, it does not provide exact values. Do not have your code rely on this return value.
+ */
+ @Override
+ public long getLength() throws IOException {
+ Text startRow = range.isInfiniteStartKey() ? new Text(new byte[] {Byte.MIN_VALUE}) : range.getStartKey().getRow();
+ Text stopRow = range.isInfiniteStopKey() ? new Text(new byte[] {Byte.MAX_VALUE}) : range.getEndKey().getRow();
+ int maxCommon = Math.min(7, Math.min(startRow.getLength(), stopRow.getLength()));
+ long diff = 0;
+
+ byte[] start = startRow.getBytes();
+ byte[] stop = stopRow.getBytes();
+ for (int i = 0; i < maxCommon; ++i) {
+ diff |= 0xff & (start[i] ^ stop[i]);
+ diff <<= Byte.SIZE;
+ }
+
+ if (startRow.getLength() != stopRow.getLength())
+ diff |= 0xff;
+
+ return diff + 1;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException {
+ return locations;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ range.readFields(in);
+ int numLocs = in.readInt();
+ locations = new String[numLocs];
+ for (int i = 0; i < numLocs; ++i)
+ locations[i] = in.readUTF();
+
+ if (in.readBoolean()) {
+ isolatedScan = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ offline = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ localIterators = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ mockInstance = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ int numColumns = in.readInt();
+ List<String> columns = new ArrayList<String>(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+ columns.add(in.readUTF());
+ }
+
+ fetchedColumns = InputConfigurator.deserializeFetchedColumns(columns);
+ }
+
+ if (in.readBoolean()) {
+ String strAuths = in.readUTF();
+ auths = new Authorizations(strAuths.getBytes(Charset.forName("UTF-8")));
+ }
+
+ if (in.readBoolean()) {
+ principal = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ String tokenClass = in.readUTF();
+ byte[] base64TokenBytes = in.readUTF().getBytes(Charset.forName("UTF-8"));
+ byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
+
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ if (in.readBoolean()) {
+ instanceName = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ zooKeepers = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ level = Level.toLevel(in.readInt());
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ range.write(out);
+ out.writeInt(locations.length);
+ for (int i = 0; i < locations.length; ++i)
+ out.writeUTF(locations[i]);
+
+ out.writeBoolean(null != isolatedScan);
+ if (null != isolatedScan) {
+ out.writeBoolean(isolatedScan);
+ }
+
+ out.writeBoolean(null != offline);
+ if (null != offline) {
+ out.writeBoolean(offline);
+ }
+
+ out.writeBoolean(null != localIterators);
+ if (null != localIterators) {
+ out.writeBoolean(localIterators);
+ }
+
+ out.writeBoolean(null != mockInstance);
+ if (null != mockInstance) {
+ out.writeBoolean(mockInstance);
+ }
+
+ out.writeBoolean(null != fetchedColumns);
+ if (null != fetchedColumns) {
+ String[] cols = InputConfigurator.serializeColumns(fetchedColumns);
+ out.writeInt(cols.length);
+ for (String col : cols) {
+ out.writeUTF(col);
+ }
+ }
+
+ out.writeBoolean(null != auths);
+ if (null != auths) {
+ out.writeUTF(auths.serialize());
+ }
+
+ out.writeBoolean(null != principal);
+ if (null != principal) {
+ out.writeUTF(principal);
+ }
+
+ out.writeBoolean(null != token);
+ if (null != token) {
+ out.writeUTF(token.getClass().getCanonicalName());
+ try {
+ out.writeUTF(CredentialHelper.tokenAsBase64(token));
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ out.writeBoolean(null != instanceName);
+ if (null != instanceName) {
+ out.writeUTF(instanceName);
+ }
+
+ out.writeBoolean(null != zooKeepers);
+ if (null != zooKeepers) {
+ out.writeUTF(zooKeepers);
+ }
+
+ out.writeBoolean(null != level);
+ if (null != level) {
+ out.writeInt(level.toInt());
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ sb.append("Range: ").append(range);
+ sb.append(" Locations: ").append(Arrays.asList(locations));
+ sb.append(" Table: ").append(table);
+ sb.append(" InstanceName: ").append(instanceName);
+ sb.append(" zooKeepers: ").append(zooKeepers);
+ sb.append(" principal: ").append(principal);
+ sb.append(" authenticationToken: ").append(token);
+ sb.append(" Authorizations: ").append(auths);
+ sb.append(" offlineScan: ").append(offline);
+ sb.append(" mockInstance: ").append(mockInstance);
+ sb.append(" isolatedScan: ").append(isolatedScan);
+ sb.append(" localIterators: ").append(localIterators);
+ sb.append(" fetchColumns: ").append(fetchedColumns);
+ sb.append(" iterators: ").append(iterators);
+ sb.append(" logLevel: ").append(level);
+ return sb.toString();
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public Instance getInstance() {
+ if (null == instanceName) {
+ return null;
+ }
+
+ if (isMockInstance()) {
+ return new MockInstance(getInstanceName());
+ }
+
+ if (null == zooKeepers) {
+ return null;
+ }
-
++
+ ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+
+ // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
+ // when it's present
+ AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+ zki.setConfiguration(xmlConfig);
+
+ return zki;
+ }
+
+ public String getInstanceName() {
+ return instanceName;
+ }
+
+ public void setInstanceName(String instanceName) {
+ this.instanceName = instanceName;
+ }
+
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ public void setZooKeepers(String zooKeepers) {
+ this.zooKeepers = zooKeepers;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ public void setToken(AuthenticationToken token) {
+ this.token = token;
+ ;
+ }
+
+ public Boolean isOffline() {
+ return offline;
+ }
+
+ public void setOffline(Boolean offline) {
+ this.offline = offline;
+ }
+
+ public void setLocations(String[] locations) {
+ this.locations = locations;
+ }
+
+ public Boolean isMockInstance() {
+ return mockInstance;
+ }
+
+ public void setMockInstance(Boolean mockInstance) {
+ this.mockInstance = mockInstance;
+ }
+
+ public Boolean isIsolatedScan() {
+ return isolatedScan;
+ }
+
+ public void setIsolatedScan(Boolean isolatedScan) {
+ this.isolatedScan = isolatedScan;
+ }
+
+ public Authorizations getAuths() {
+ return auths;
+ }
+
+ public void setAuths(Authorizations auths) {
+ this.auths = auths;
+ }
+
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ public Boolean usesLocalIterators() {
+ return localIterators;
+ }
+
+ public void setUsesLocalIterators(Boolean localIterators) {
+ this.localIterators = localIterators;
+ }
+
+ public Set<Pair<Text,Text>> getFetchedColumns() {
+ return fetchedColumns;
+ }
+
+ public void setFetchedColumns(Set<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = fetchedColumns;
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+
+ public void setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ }
+
+ public Level getLogLevel() {
+ return level;
+ }
+
+ public void setLogLevel(Level level) {
+ this.level = level;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a147acdd/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 69a483f,0000000..a9d7153
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -1,265 -1,0 +1,265 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.test.continuous.ContinuousWalk.BadChecksumException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
+ */
+
+public class ContinuousVerify extends Configured implements Tool {
+ private static final Logger log = Logger.getLogger(ContinuousVerify.class);
-
++
+ // work around hadoop-1/hadoop-2 runtime incompatibility
+ static private Method INCREMENT;
+ static {
+ try {
+ INCREMENT = Counter.class.getMethod("increment", Long.TYPE);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
-
++
+ static void increment(Object obj) {
+ try {
+ INCREMENT.invoke(obj, 1L);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
-
++
+ public static final VLongWritable DEF = new VLongWritable(-1);
-
++
+ public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
-
++
+ private LongWritable row = new LongWritable();
+ private LongWritable ref = new LongWritable();
+ private VLongWritable vrow = new VLongWritable();
+
+ private long corrupt = 0;
-
++
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+ long r = Long.parseLong(key.getRow().toString(), 16);
+ if (r < 0)
+ throw new IllegalArgumentException();
-
++
+ try {
+ ContinuousWalk.validate(key, data);
+ } catch (BadChecksumException bce) {
+ increment(context.getCounter(Counts.CORRUPT));
+ if (corrupt < 1000) {
+ System.out.println("ERROR Bad checksum : " + key);
+ } else if (corrupt == 1000) {
+ System.out.println("Too many bad checksums, not printing anymore!");
+ }
+ corrupt++;
+ return;
+ }
-
++
+ row.set(r);
-
++
+ context.write(row, DEF);
+ byte[] val = data.get();
-
++
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ ref.set(Long.parseLong(new String(val, offset, 16), 16));
+ vrow.set(r);
+ context.write(ref, vrow);
+ }
+ }
+ }
-
++
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ }
-
++
+ public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+ private ArrayList<Long> refs = new ArrayList<Long>();
-
++
+ @Override
+ public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
-
++
+ int defCount = 0;
-
++
+ refs.clear();
+ for (VLongWritable type : values) {
+ if (type.get() == -1) {
+ defCount++;
+ } else {
+ refs.add(type.get());
+ }
+ }
-
++
+ if (defCount == 0 && refs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ String comma = "";
+ for (Long ref : refs) {
+ sb.append(comma);
+ comma = ",";
+ sb.append(new String(ContinuousIngest.genRow(ref)));
+ }
-
++
+ context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
+ increment(context.getCounter(Counts.UNDEFINED));
-
++
+ } else if (defCount > 0 && refs.size() == 0) {
+ increment(context.getCounter(Counts.UNREFERENCED));
+ } else {
+ increment(context.getCounter(Counts.REFERENCED));
+ }
-
++
+ }
+ }
-
++
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names = "--output", description = "location in HDFS to store the results; must not exist", required = true)
+ String outputDir = "/tmp/continuousVerify";
-
++
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
-
++
+ @Parameter(names = "--reducers", description = "the number of reducers to use", required = true, validateWith = PositiveInteger.class)
+ int reducers = 0;
-
++
+ @Parameter(names = "--offline", description = "perform the verification directly on the files while the table is offline")
+ boolean scanOffline = false;
-
++
+ @Parameter(names = "--sitefile", description = "location of accumulo-site.xml in HDFS", required = true)
+ String siteFile;
-
++
+ public Opts() {
+ super("ci");
+ }
+ }
-
++
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(this.getClass().getName(), args);
-
++
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
-
++
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ String clone = opts.getTableName();
+ Connector conn = null;
+ if (opts.scanOffline) {
+ Random random = new Random();
+ clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
+ conn = opts.getConnector();
+ conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
+ conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
+ }
-
++
+ // set up ranges
+ try {
+ Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
-
++
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
-
++
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(opts.reducers);
-
++
+ job.setOutputFormatClass(TextOutputFormat.class);
-
++
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);
-
++
+ TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));
-
++
+ Path sitePath = new Path(opts.siteFile);
+ Path siteParentPath = sitePath.getParent();
+ if (null == siteParentPath) {
+ siteParentPath = new Path("/");
+ }
-
++
+ URI siteUri = new URI("hdfs://" + opts.siteFile);
-
++
+ log.info("Adding " + siteUri + " to DistributedCache");
-
++
+ // Make sure that accumulo-site.xml is available for mappers running offline scans
+ // as they need to correctly choose instance.dfs.dir for the installation
+ DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
-
++
+ job.waitForCompletion(true);
-
++
+ if (opts.scanOffline) {
+ conn.tableOperations().delete(clone);
+ }
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
-
++
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
[12/15] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Conflicts:
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a147acdd
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a147acdd
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a147acdd
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: a147acdd600f2fce2a78c9845d00ac3779185707
Parents: 00bd820 9cf94f9
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:31:48 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:31:48 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ++++++++++++++++++
.../test/continuous/ContinuousVerify.java | 82 +-
3 files changed, 1676 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a147acdd/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 592cde6,0000000..f7b2263
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -1,442 -1,0 +1,442 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mapreduce.lib.util.InputConfigurator;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.log4j.Level;
+
+/**
+ * The Class RangeInputSplit. Encapsulates an Accumulo range for use in Map Reduce jobs.
+ */
+public class RangeInputSplit extends InputSplit implements Writable {
+ private Range range;
+ private String[] locations;
+ private String table, instanceName, zooKeepers, principal;
+ private AuthenticationToken token;
+ private Boolean offline, mockInstance, isolatedScan, localIterators;
+ private Authorizations auths;
+ private Set<Pair<Text,Text>> fetchedColumns;
+ private List<IteratorSetting> iterators;
+ private Level level;
+
+ public RangeInputSplit() {
+ range = new Range();
+ locations = new String[0];
+ }
+
+ public RangeInputSplit(Range range, String[] locations) {
+ this.range = range;
+ this.locations = locations;
+ }
+
+ public Range getRange() {
+ return range;
+ }
+
+ private static byte[] extractBytes(ByteSequence seq, int numBytes) {
+ byte[] bytes = new byte[numBytes + 1];
+ bytes[0] = 0;
+ for (int i = 0; i < numBytes; i++) {
+ if (i >= seq.length())
+ bytes[i + 1] = 0;
+ else
+ bytes[i + 1] = seq.byteAt(i);
+ }
+ return bytes;
+ }
+
+ public static float getProgress(ByteSequence start, ByteSequence end, ByteSequence position) {
+ int maxDepth = Math.min(Math.max(end.length(), start.length()), position.length());
+ BigInteger startBI = new BigInteger(extractBytes(start, maxDepth));
+ BigInteger endBI = new BigInteger(extractBytes(end, maxDepth));
+ BigInteger positionBI = new BigInteger(extractBytes(position, maxDepth));
+ return (float) (positionBI.subtract(startBI).doubleValue() / endBI.subtract(startBI).doubleValue());
+ }
+
+ public float getProgress(Key currentKey) {
+ if (currentKey == null)
+ return 0f;
+ if (range.getStartKey() != null && range.getEndKey() != null) {
+ if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW) != 0) {
+ // just look at the row progress
+ return getProgress(range.getStartKey().getRowData(), range.getEndKey().getRowData(), currentKey.getRowData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM) != 0) {
+ // just look at the column family progress
+ return getProgress(range.getStartKey().getColumnFamilyData(), range.getEndKey().getColumnFamilyData(), currentKey.getColumnFamilyData());
+ } else if (range.getStartKey().compareTo(range.getEndKey(), PartialKey.ROW_COLFAM_COLQUAL) != 0) {
+ // just look at the column qualifier progress
+ return getProgress(range.getStartKey().getColumnQualifierData(), range.getEndKey().getColumnQualifierData(), currentKey.getColumnQualifierData());
+ }
+ }
+ // if we can't figure it out, then claim no progress
+ return 0f;
+ }
+
+ /**
+ * This implementation of length is only an estimate, it does not provide exact values. Do not have your code rely on this return value.
+ */
+ @Override
+ public long getLength() throws IOException {
+ Text startRow = range.isInfiniteStartKey() ? new Text(new byte[] {Byte.MIN_VALUE}) : range.getStartKey().getRow();
+ Text stopRow = range.isInfiniteStopKey() ? new Text(new byte[] {Byte.MAX_VALUE}) : range.getEndKey().getRow();
+ int maxCommon = Math.min(7, Math.min(startRow.getLength(), stopRow.getLength()));
+ long diff = 0;
+
+ byte[] start = startRow.getBytes();
+ byte[] stop = stopRow.getBytes();
+ for (int i = 0; i < maxCommon; ++i) {
+ diff |= 0xff & (start[i] ^ stop[i]);
+ diff <<= Byte.SIZE;
+ }
+
+ if (startRow.getLength() != stopRow.getLength())
+ diff |= 0xff;
+
+ return diff + 1;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException {
+ return locations;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ range.readFields(in);
+ int numLocs = in.readInt();
+ locations = new String[numLocs];
+ for (int i = 0; i < numLocs; ++i)
+ locations[i] = in.readUTF();
+
+ if (in.readBoolean()) {
+ isolatedScan = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ offline = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ localIterators = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ mockInstance = in.readBoolean();
+ }
+
+ if (in.readBoolean()) {
+ int numColumns = in.readInt();
+ List<String> columns = new ArrayList<String>(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+ columns.add(in.readUTF());
+ }
+
+ fetchedColumns = InputConfigurator.deserializeFetchedColumns(columns);
+ }
+
+ if (in.readBoolean()) {
+ String strAuths = in.readUTF();
+ auths = new Authorizations(strAuths.getBytes(Charset.forName("UTF-8")));
+ }
+
+ if (in.readBoolean()) {
+ principal = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ String tokenClass = in.readUTF();
+ byte[] base64TokenBytes = in.readUTF().getBytes(Charset.forName("UTF-8"));
+ byte[] tokenBytes = Base64.decodeBase64(base64TokenBytes);
+
+ try {
+ token = CredentialHelper.extractToken(tokenClass, tokenBytes);
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ if (in.readBoolean()) {
+ instanceName = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ zooKeepers = in.readUTF();
+ }
+
+ if (in.readBoolean()) {
+ level = Level.toLevel(in.readInt());
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ range.write(out);
+ out.writeInt(locations.length);
+ for (int i = 0; i < locations.length; ++i)
+ out.writeUTF(locations[i]);
+
+ out.writeBoolean(null != isolatedScan);
+ if (null != isolatedScan) {
+ out.writeBoolean(isolatedScan);
+ }
+
+ out.writeBoolean(null != offline);
+ if (null != offline) {
+ out.writeBoolean(offline);
+ }
+
+ out.writeBoolean(null != localIterators);
+ if (null != localIterators) {
+ out.writeBoolean(localIterators);
+ }
+
+ out.writeBoolean(null != mockInstance);
+ if (null != mockInstance) {
+ out.writeBoolean(mockInstance);
+ }
+
+ out.writeBoolean(null != fetchedColumns);
+ if (null != fetchedColumns) {
+ String[] cols = InputConfigurator.serializeColumns(fetchedColumns);
+ out.writeInt(cols.length);
+ for (String col : cols) {
+ out.writeUTF(col);
+ }
+ }
+
+ out.writeBoolean(null != auths);
+ if (null != auths) {
+ out.writeUTF(auths.serialize());
+ }
+
+ out.writeBoolean(null != principal);
+ if (null != principal) {
+ out.writeUTF(principal);
+ }
+
+ out.writeBoolean(null != token);
+ if (null != token) {
+ out.writeUTF(token.getClass().getCanonicalName());
+ try {
+ out.writeUTF(CredentialHelper.tokenAsBase64(token));
+ } catch (AccumuloSecurityException e) {
+ throw new IOException(e);
+ }
+ }
+
+ out.writeBoolean(null != instanceName);
+ if (null != instanceName) {
+ out.writeUTF(instanceName);
+ }
+
+ out.writeBoolean(null != zooKeepers);
+ if (null != zooKeepers) {
+ out.writeUTF(zooKeepers);
+ }
+
+ out.writeBoolean(null != level);
+ if (null != level) {
+ out.writeInt(level.toInt());
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(256);
+ sb.append("Range: ").append(range);
+ sb.append(" Locations: ").append(Arrays.asList(locations));
+ sb.append(" Table: ").append(table);
+ sb.append(" InstanceName: ").append(instanceName);
+ sb.append(" zooKeepers: ").append(zooKeepers);
+ sb.append(" principal: ").append(principal);
+ sb.append(" authenticationToken: ").append(token);
+ sb.append(" Authorizations: ").append(auths);
+ sb.append(" offlineScan: ").append(offline);
+ sb.append(" mockInstance: ").append(mockInstance);
+ sb.append(" isolatedScan: ").append(isolatedScan);
+ sb.append(" localIterators: ").append(localIterators);
+ sb.append(" fetchColumns: ").append(fetchedColumns);
+ sb.append(" iterators: ").append(iterators);
+ sb.append(" logLevel: ").append(level);
+ return sb.toString();
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public Instance getInstance() {
+ if (null == instanceName) {
+ return null;
+ }
+
+ if (isMockInstance()) {
+ return new MockInstance(getInstanceName());
+ }
+
+ if (null == zooKeepers) {
+ return null;
+ }
-
++
+ ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+
+ // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
+ // when it's present
+ AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+ zki.setConfiguration(xmlConfig);
+
+ return zki;
+ }
+
+ public String getInstanceName() {
+ return instanceName;
+ }
+
+ public void setInstanceName(String instanceName) {
+ this.instanceName = instanceName;
+ }
+
+ public String getZooKeepers() {
+ return zooKeepers;
+ }
+
+ public void setZooKeepers(String zooKeepers) {
+ this.zooKeepers = zooKeepers;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public AuthenticationToken getToken() {
+ return token;
+ }
+
+ public void setToken(AuthenticationToken token) {
+ this.token = token;
+ ;
+ }
+
+ public Boolean isOffline() {
+ return offline;
+ }
+
+ public void setOffline(Boolean offline) {
+ this.offline = offline;
+ }
+
+ public void setLocations(String[] locations) {
+ this.locations = locations;
+ }
+
+ public Boolean isMockInstance() {
+ return mockInstance;
+ }
+
+ public void setMockInstance(Boolean mockInstance) {
+ this.mockInstance = mockInstance;
+ }
+
+ public Boolean isIsolatedScan() {
+ return isolatedScan;
+ }
+
+ public void setIsolatedScan(Boolean isolatedScan) {
+ this.isolatedScan = isolatedScan;
+ }
+
+ public Authorizations getAuths() {
+ return auths;
+ }
+
+ public void setAuths(Authorizations auths) {
+ this.auths = auths;
+ }
+
+ public void setRange(Range range) {
+ this.range = range;
+ }
+
+ public Boolean usesLocalIterators() {
+ return localIterators;
+ }
+
+ public void setUsesLocalIterators(Boolean localIterators) {
+ this.localIterators = localIterators;
+ }
+
+ public Set<Pair<Text,Text>> getFetchedColumns() {
+ return fetchedColumns;
+ }
+
+ public void setFetchedColumns(Set<Pair<Text,Text>> fetchedColumns) {
+ this.fetchedColumns = fetchedColumns;
+ }
+
+ public List<IteratorSetting> getIterators() {
+ return iterators;
+ }
+
+ public void setIterators(List<IteratorSetting> iterators) {
+ this.iterators = iterators;
+ }
+
+ public Level getLogLevel() {
+ return level;
+ }
+
+ public void setLogLevel(Level level) {
+ this.level = level;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a147acdd/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 69a483f,0000000..a9d7153
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -1,265 -1,0 +1,265 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.continuous;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.test.continuous.ContinuousWalk.BadChecksumException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VLongWritable;
+import org.apache.hadoop.mapred.Counters.Counter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
+ */
+
+public class ContinuousVerify extends Configured implements Tool {
+ private static final Logger log = Logger.getLogger(ContinuousVerify.class);
-
++
+ // work around hadoop-1/hadoop-2 runtime incompatibility
+ static private Method INCREMENT;
+ static {
+ try {
+ INCREMENT = Counter.class.getMethod("increment", Long.TYPE);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
-
++
+ static void increment(Object obj) {
+ try {
+ INCREMENT.invoke(obj, 1L);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
-
++
+ public static final VLongWritable DEF = new VLongWritable(-1);
-
++
+ public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
-
++
+ private LongWritable row = new LongWritable();
+ private LongWritable ref = new LongWritable();
+ private VLongWritable vrow = new VLongWritable();
+
+ private long corrupt = 0;
-
++
+ @Override
+ public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
+ long r = Long.parseLong(key.getRow().toString(), 16);
+ if (r < 0)
+ throw new IllegalArgumentException();
-
++
+ try {
+ ContinuousWalk.validate(key, data);
+ } catch (BadChecksumException bce) {
+ increment(context.getCounter(Counts.CORRUPT));
+ if (corrupt < 1000) {
+ System.out.println("ERROR Bad checksum : " + key);
+ } else if (corrupt == 1000) {
+ System.out.println("Too many bad checksums, not printing anymore!");
+ }
+ corrupt++;
+ return;
+ }
-
++
+ row.set(r);
-
++
+ context.write(row, DEF);
+ byte[] val = data.get();
-
++
+ int offset = ContinuousWalk.getPrevRowOffset(val);
+ if (offset > 0) {
+ ref.set(Long.parseLong(new String(val, offset, 16), 16));
+ vrow.set(r);
+ context.write(ref, vrow);
+ }
+ }
+ }
-
++
+ public static enum Counts {
+ UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
+ }
-
++
+ public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
+ private ArrayList<Long> refs = new ArrayList<Long>();
-
++
+ @Override
+ public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
-
++
+ int defCount = 0;
-
++
+ refs.clear();
+ for (VLongWritable type : values) {
+ if (type.get() == -1) {
+ defCount++;
+ } else {
+ refs.add(type.get());
+ }
+ }
-
++
+ if (defCount == 0 && refs.size() > 0) {
+ StringBuilder sb = new StringBuilder();
+ String comma = "";
+ for (Long ref : refs) {
+ sb.append(comma);
+ comma = ",";
+ sb.append(new String(ContinuousIngest.genRow(ref)));
+ }
-
++
+ context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
+ increment(context.getCounter(Counts.UNDEFINED));
-
++
+ } else if (defCount > 0 && refs.size() == 0) {
+ increment(context.getCounter(Counts.UNREFERENCED));
+ } else {
+ increment(context.getCounter(Counts.REFERENCED));
+ }
-
++
+ }
+ }
-
++
+ static class Opts extends ClientOnDefaultTable {
+ @Parameter(names = "--output", description = "location in HDFS to store the results; must not exist", required = true)
+ String outputDir = "/tmp/continuousVerify";
-
++
+ @Parameter(names = "--maxMappers", description = "the maximum number of mappers to use", required = true, validateWith = PositiveInteger.class)
+ int maxMaps = 0;
-
++
+ @Parameter(names = "--reducers", description = "the number of reducers to use", required = true, validateWith = PositiveInteger.class)
+ int reducers = 0;
-
++
+ @Parameter(names = "--offline", description = "perform the verification directly on the files while the table is offline")
+ boolean scanOffline = false;
-
++
+ @Parameter(names = "--sitefile", description = "location of accumulo-site.xml in HDFS", required = true)
+ String siteFile;
-
++
+ public Opts() {
+ super("ci");
+ }
+ }
-
++
+ @Override
+ public int run(String[] args) throws Exception {
+ Opts opts = new Opts();
+ opts.parseArgs(this.getClass().getName(), args);
-
++
+ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+ job.setJarByClass(this.getClass());
-
++
+ job.setInputFormatClass(AccumuloInputFormat.class);
+ opts.setAccumuloConfigs(job);
+
+ String clone = opts.getTableName();
+ Connector conn = null;
+ if (opts.scanOffline) {
+ Random random = new Random();
+ clone = opts.getTableName() + "_" + String.format("%016x", (random.nextLong() & 0x7fffffffffffffffl));
+ conn = opts.getConnector();
+ conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap<String,String>(), new HashSet<String>());
+ conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
+ }
-
++
+ // set up ranges
+ try {
+ Set<Range> ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
+ AccumuloInputFormat.setRanges(job, ranges);
+ AccumuloInputFormat.setAutoAdjustRanges(job, false);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
-
++
+ job.setMapperClass(CMapper.class);
+ job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputValueClass(VLongWritable.class);
-
++
+ job.setReducerClass(CReducer.class);
+ job.setNumReduceTasks(opts.reducers);
-
++
+ job.setOutputFormatClass(TextOutputFormat.class);
-
++
+ job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", opts.scanOffline);
-
++
+ TextOutputFormat.setOutputPath(job, new Path(opts.outputDir));
-
++
+ Path sitePath = new Path(opts.siteFile);
+ Path siteParentPath = sitePath.getParent();
+ if (null == siteParentPath) {
+ siteParentPath = new Path("/");
+ }
-
++
+ URI siteUri = new URI("hdfs://" + opts.siteFile);
-
++
+ log.info("Adding " + siteUri + " to DistributedCache");
-
++
+ // Make sure that accumulo-site.xml is available for mappers running offline scans
+ // as they need to correctly choose instance.dfs.dir for the installation
+ DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
-
++
+ job.waitForCompletion(true);
-
++
+ if (opts.scanOffline) {
+ conn.tableOperations().delete(clone);
+ }
+ opts.stopTracing();
+ return job.isSuccessful() ? 0 : 1;
+ }
-
++
+ /**
+ *
+ * @param args
+ * instanceName zookeepers username password table columns outputpath
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(CachedConfiguration.getInstance(), new ContinuousVerify(), args);
+ if (res != 0)
+ System.exit(res);
+ }
+}
[05/15] git commit: ACCUMULO-2234 Fix up formatting for madrob.
Posted by el...@apache.org.
ACCUMULO-2234 Fix up formatting for madrob.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9cf94f93
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9cf94f93
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9cf94f93
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 9cf94f93685faaccc927e148791d50570bfb2f30
Parents: de7d198
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:19:30 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:19:30 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/InputFormatBase.java | 2 +-
.../test/continuous/ContinuousVerify.java | 74 ++++++++++----------
2 files changed, 38 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf94f93/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 9c23246..705ddbd 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -737,7 +737,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
protected static Instance getInstance(Configuration conf) {
if (conf.getBoolean(MOCK, false))
return new MockInstance(conf.get(INSTANCE_NAME));
-
+
ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
// Wrap the DefaultConfiguration with a SiteConfiguration
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9cf94f93/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index 6546eea..4a5c0cf 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -55,22 +55,22 @@ import org.apache.log4j.Logger;
public class ContinuousVerify extends Configured implements Tool {
private static final Logger log = Logger.getLogger(ContinuousVerify.class);
-
+
public static final VLongWritable DEF = new VLongWritable(-1);
-
+
public static class CMapper extends Mapper<Key,Value,LongWritable,VLongWritable> {
-
+
private LongWritable row = new LongWritable();
private LongWritable ref = new LongWritable();
private VLongWritable vrow = new VLongWritable();
-
+
private long corrupt = 0;
-
+
public void map(Key key, Value data, Context context) throws IOException, InterruptedException {
long r = Long.parseLong(key.getRow().toString(), 16);
if (r < 0)
throw new IllegalArgumentException();
-
+
try {
ContinuousWalk.validate(key, data);
} catch (BadChecksumException bce) {
@@ -83,12 +83,12 @@ public class ContinuousVerify extends Configured implements Tool {
corrupt++;
return;
}
-
+
row.set(r);
-
+
context.write(row, DEF);
byte[] val = data.get();
-
+
int offset = ContinuousWalk.getPrevRowOffset(val);
if (offset > 0) {
ref.set(Long.parseLong(new String(val, offset, 16), 16));
@@ -97,18 +97,18 @@ public class ContinuousVerify extends Configured implements Tool {
}
}
}
-
+
public static enum Counts {
UNREFERENCED, UNDEFINED, REFERENCED, CORRUPT
}
-
+
public static class CReducer extends Reducer<LongWritable,VLongWritable,Text,Text> {
private ArrayList<Long> refs = new ArrayList<Long>();
-
+
public void reduce(LongWritable key, Iterable<VLongWritable> values, Context context) throws IOException, InterruptedException {
-
+
int defCount = 0;
-
+
refs.clear();
for (VLongWritable type : values) {
if (type.get() == -1) {
@@ -117,7 +117,7 @@ public class ContinuousVerify extends Configured implements Tool {
refs.add(type.get());
}
}
-
+
if (defCount == 0 && refs.size() > 0) {
StringBuilder sb = new StringBuilder();
String comma = "";
@@ -126,25 +126,25 @@ public class ContinuousVerify extends Configured implements Tool {
comma = ",";
sb.append(new String(ContinuousIngest.genRow(ref)));
}
-
+
context.write(new Text(ContinuousIngest.genRow(key.get())), new Text(sb.toString()));
context.getCounter(Counts.UNDEFINED).increment(1);
-
+
} else if (defCount > 0 && refs.size() == 0) {
context.getCounter(Counts.UNREFERENCED).increment(1);
} else {
context.getCounter(Counts.REFERENCED).increment(1);
}
-
+
}
}
-
+
@Override
public int run(String[] args) throws Exception {
-
+
String auths = "";
ArrayList<String> argsList = new ArrayList<String>();
-
+
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--auths")) {
auths = args[++i];
@@ -152,14 +152,14 @@ public class ContinuousVerify extends Configured implements Tool {
argsList.add(args[i]);
}
}
-
+
args = argsList.toArray(new String[0]);
if (args.length != 10) {
throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
+ " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
}
-
+
String instance = args[0];
String zookeepers = args[1];
String user = args[2];
@@ -170,10 +170,10 @@ public class ContinuousVerify extends Configured implements Tool {
String reducers = args[7];
boolean scanOffline = Boolean.parseBoolean(args[8]);
String siteFile = args[9];
-
+
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
-
+
String clone = table;
Connector conn = null;
if (scanOffline) {
@@ -205,43 +205,43 @@ public class ContinuousVerify extends Configured implements Tool {
} catch (Exception e) {
throw new IOException(e);
}
-
+
job.setMapperClass(CMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(VLongWritable.class);
-
+
job.setReducerClass(CReducer.class);
job.setNumReduceTasks(Integer.parseInt(reducers));
-
+
job.setOutputFormatClass(TextOutputFormat.class);
-
+
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
-
+
Path sitePath = new Path(siteFile);
Path siteParentPath = sitePath.getParent();
if (null == siteParentPath) {
siteParentPath = new Path("/");
}
-
+
URI siteUri = new URI("hdfs://" + siteFile);
-
+
log.info("Adding " + siteUri + " to DistributedCache");
-
+
// Make sure that accumulo-site.xml is available for mappers running offline scans
// as they need to correctly choose instance.dfs.dir for the installation
DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
TextOutputFormat.setOutputPath(job, new Path(outputdir));
-
+
job.waitForCompletion(true);
-
+
if (scanOffline) {
conn.tableOperations().delete(clone);
}
return job.isSuccessful() ? 0 : 1;
}
-
+
/**
*
* @param args
@@ -253,4 +253,4 @@ public class ContinuousVerify extends Configured implements Tool {
if (res != 0)
System.exit(res);
}
-}
\ No newline at end of file
+}
[14/15] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9440d15c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9440d15c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9440d15c
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 9440d15cbe0a1639c509bb37d2e30830efdfddc9
Parents: ecdd852 a147acd
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 13:05:00 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 13:05:00 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../core/util/shell/commands/DUCommand.java | 4 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ++++++++++++++++++
.../test/continuous/ContinuousVerify.java | 80 +-
.../org/apache/accumulo/test/ShellServerIT.java | 112 +-
5 files changed, 1734 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 85a0104,f7b2263..831d10b
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -369,8 -324,8 +369,8 @@@ public class RangeInputSplit extends In
if (null == zooKeepers) {
return null;
}
-
+
- ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+ ZooKeeperInstance zki = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(getInstanceName()).withZkHosts(getZooKeepers()));
// Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
// when it's present
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
index a0fcc10,ca80e37..660ec6c
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
@@@ -67,18 -52,25 +67,18 @@@ public class DUCommand extends Command
}
}
}
-
+
// If we didn't get any tables, and we have a table selected, add the current table
- if (tablesToFlush.isEmpty() && !shellState.getTableName().isEmpty()) {
- tablesToFlush.add(shellState.getTableName());
+ if (tables.isEmpty() && !shellState.getTableName().isEmpty()) {
+ tables.add(shellState.getTableName());
}
-
+
try {
- final AccumuloConfiguration acuConf = new ConfigurationCopy(shellState.getConnector().instanceOperations().getSystemConfiguration());
- TableDiskUsage.printDiskUsage(acuConf, tablesToFlush, FileSystem.get(new Configuration()), shellState.getConnector(), new Printer() {
- @Override
- public void print(String line) {
- try {
- shellState.getReader().printString(line + "\n");
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }
-
- }, prettyPrint);
+ String valueFormat = prettyPrint ? "%9s" : "%,24d";
+ for (DiskUsage usage : shellState.getConnector().tableOperations().getDiskUsage(tables)) {
+ Object value = prettyPrint ? NumUtil.bigNumberForSize(usage.getUsage()) : usage.getUsage();
+ shellState.getReader().println(String.format(valueFormat + " %s", value, usage.getTables()));
+ }
} catch (Exception ex) {
throw new RuntimeException(ex);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 09ebdd1,a9d7153..fbcb6fa
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -188,10 -188,9 +188,10 @@@ public class ContinuousVerify extends C
Opts opts = new Opts();
opts.parseArgs(this.getClass().getName(), args);
+ @SuppressWarnings("deprecation")
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
-
+
job.setInputFormatClass(AccumuloInputFormat.class);
opts.setAccumuloConfigs(job);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index 2a06143,0000000..81f9ef2
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@@ -1,1048 -1,0 +1,1050 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.util.Map.Entry;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ShellServerIT extends SimpleMacIT {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ public static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() throws IOException {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+
+ public static TestOutputStream output;
+ public static StringInputStream input;
+ public static Shell shell;
+ private static Process traceProcess;
+
+ static String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ static String exec(String cmd, boolean expectGoodExit) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit("", true);
+ else
+ assertBadExit("", true);
+ return result;
+ }
+
+ static String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true);
+ }
+
+ static String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit(expectString, stringPresent);
+ else
+ assertBadExit(expectString, stringPresent);
+ return result;
+ }
+
+ static void assertGoodExit(String s, boolean stringPresent) {
+ Shell.log.info(output.get());
+ assertEquals(0, shell.getExitCode());
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
+ static void assertBadExit(String s, boolean stringPresent) {
+ Shell.log.debug(output.get());
+ assertTrue(shell.getExitCode() > 0);
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // history file is updated in $HOME
+ System.setProperty("HOME", getFolder().getAbsolutePath());
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
+ shell = new Shell(new ConsoleReader(input, output));
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(),
+ "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+
+ traceProcess = getStaticCluster().exec(TraceServer.class);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ traceProcess.destroy();
+ }
+
+ @After
+ public void deleteTables() throws Exception {
+ Connector c = getConnector();
+ for (String table : c.tableOperations().list()) {
+ if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace"))
+ try {
+ c.tableOperations().delete(table);
+ } catch (TableNotFoundException e) {
+ // don't care
+ }
+ }
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void exporttableImporttable() throws Exception {
+ // exporttable / importtable
+ exec("createtable t -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t t -s table.split.threshold=345M", true);
+ exec("offline t", true);
+ String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString();
+ exec("exporttable -t t " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable t2 " + import_, true);
+ exec("config -t t2 -np", true, "345M", true);
+ exec("getsplits -t t2", true, "row5", true);
+ exec("constraint --list -t t2", true, "VisibilityConstraint=2", true);
+ exec("onlinetable t", true);
+ exec("deletetable -f t", true);
+ exec("deletetable -f t2", true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void setscaniterDeletescaniter() throws Exception {
+ // setscaniter, deletescaniter
+ exec("createtable t");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING");
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f t");
+
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder());
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void egrep() throws Exception {
+ // egrep
+ exec("createtable t");
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void du() throws Exception {
++ // create and delete a table so we get out of a table context in the shell
++ exec("createtable du_test_table", true);
++ exec("deletetable -f du_test_table", true);
++
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
-
++
+ output.clear();
+ exec("createtable t");
+ make10();
+ exec("flush -t t -w");
+ exec("du t", true, " [t]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*[1-9][0-9][0-9]\\s\\[t\\]\\n"));
+ exec("deletetable -f t");
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void user() throws Exception {
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ input.set("secret\nsecret\n");
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (" + MetadataTable.NAME + "): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ input.set("secret\nsecret\n");
+ exec("user xyzzy", true);
+ exec("createtable t", true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f t", true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void iter() throws Exception {
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable t");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n name", false);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f t");
+ // list iter
+ exec("createtable t");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n name", false);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f t");
+
+ }
-
- @Test(timeout = 30 * 1000)
++
++ @Test(timeout = 30000)
+ public void setIterOptionPrompt() throws Exception {
+ Connector conn = getConnector();
+ String tableName = "setIterOptionPrompt";
-
++
+ exec("createtable " + tableName);
+ input.set("\n\n");
+ // Setting a non-optiondescriber with no name should fail
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", false);
-
++
+ // Name as option will work
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+
+ String expectedKey = "table.iterator.scan.cfcounter";
+ String expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ TableOperations tops = conn.tableOperations();
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
-
++
+ exec("createtable " + tableName, true);
-
++
+ input.set("customcfcounter\n\n");
-
++
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
-
++
+ exec("createtable " + tableName, true);
-
++
+ input.set("customcfcounter\nname1 value1\nname2 value2\n\n");
-
++
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name1";
+ expectedValue = "value1";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
-
++
+ exec("createtable " + tableName, true);
-
++
+ input.set("\nname1 value1.1,value1.2,value1.3\nname2 value2\n\n");
-
++
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+ expectedKey = "table.iterator.scan.cfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name1";
+ expectedValue = "value1.1,value1.2,value1.3";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ }
+
+ protected void checkTableForProperty(TableOperations tops, String tableName, String expectedKey, String expectedValue) throws Exception {
+ for (int i = 0; i < 5; i++) {
+ for (Entry<String,String> entry : tops.getProperties(tableName)) {
+ if (expectedKey.equals(entry.getKey())) {
+ assertEquals(expectedValue, entry.getValue());
+ return;
+ }
+ }
+ Thread.sleep(500);
+ }
+
+ fail("Failed to find expected property on " + tableName + ": " + expectedKey + "=" + expectedValue);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void notable() throws Exception {
+ // notable
+ exec("createtable xyzzy", true);
+ exec("scan", true, " xyzzy>", true);
+ assertTrue(output.get().contains(" xyzzy>"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" xyzzy>"));
+ exec("deletetable -f xyzzy");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue(diff >= 200);
+ assertTrue(diff < 400);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void addauths() throws Exception {
+ // addauths
+ exec("createtable xyzzy -evc");
+ exec("insert a b c d -l foo", false, "does not have authorization", true);
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f xyzzy");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ if (shell.getReader().getTerminal().isAnsiSupported()) {
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ } else {
+ exec("cls", false, "does not support");
+ exec("clear", false, "does not support");
+ }
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void testCompactions() throws IOException {
+ // compact
+ exec("createtable c");
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles();
+ // merge two files into one
+ exec("compact -t c -w");
+ assertTrue(countFiles() < oldCount);
+ exec("addsplits -t c f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
+ oldCount = countFiles();
+ // at this point there are 3 files in the default tablet
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertTrue(countFiles() == oldCount - 2);
+ exec("compact -w");
+ assertTrue(countFiles() == oldCount - 2);
+ exec("merge --all -t c");
+ exec("compact -w");
+ assertTrue(countFiles() == oldCount - 3);
+ exec("deletetable -f c");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void clonetable() throws Exception {
+ // clonetable
+ exec("createtable orig -evc");
+ exec("config -t orig -s table.split.threshold=123M", true);
+ exec("addsplits -t orig a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable orig clone");
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t clone", true, "VisibilityConstraint=2", true);
+ exec("config -t clone -np", true, "123M", true);
+ exec("getsplits -t clone", true, "a\nb\nc\n");
+ exec("deletetable -f orig");
+ exec("deletetable -f clone");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void constraint() throws Exception {
+ // constraint
+ exec("constraint -l -t " + MetadataTable.NAME + "", true, "MetadataConstraints=1", true);
+ exec("createtable c -evc");
+ exec("constraint -l -t c", true, "VisibilityConstraint=2", true);
+ exec("constraint -t c -d 2", true, "Removed constraint 2 from table c");
+ // wait for zookeeper updates to propagate
+ UtilWaitThread.sleep(1000);
+ exec("constraint -l -t c", true, "VisibilityConstraint=2", false);
+ exec("deletetable -f c");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void deletemany() throws Exception {
+ // deletemany
+ exec("createtable t");
+ make10();
+ assertEquals(10, countkeys("t"));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys("t"));
+ exec("scan -t t -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys("t"));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys("t"));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys("t"));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys("t"));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys("t"));
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void deleterows() throws Exception {
+ // deleterows
+ int base = countFiles();
+ exec("createtable t");
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t t");
+ assertEquals(base + 3, countFiles());
+ exec("deleterows -t t -b row5 -e row7", true);
+ assertEquals(base + 2, countFiles());
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void groups() throws Exception {
+ exec("createtable t");
+ exec("setgroups -t t alpha=a,b,c num=3,2,1");
+ exec("getgroups -t t", true, "alpha=a,b,c", true);
+ exec("getgroups -t t", true, "num=1,2,3", true);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void grep() throws Exception {
+ exec("createtable t", true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f t", true);
+ }
+
- @Test
- // (timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
-
- // @Test(timeout = 30 * 1000)
++ // @Test(timeout = 30000)
+ public void history() throws Exception {
+ exec("history -c", true);
+ exec("createtable unusualstring");
+ exec("deletetable -f unusualstring");
+ exec("history", true, "unusualstring", true);
+ exec("history", true, "history", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void importDirectory() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = new File(getFolder(), "import");
+ importDir.mkdir();
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = new File(getFolder(), "errors");
+ errorsDir.mkdir();
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ assertEquals(0, shell.getExitCode());
+ exec("createtable t", true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void interpreter() throws Exception {
+ exec("createtable t", true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+ UtilWaitThread.sleep(500);
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void listcompactions() throws Exception {
+ exec("createtable t", true);
+ exec("config -t t -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t t -s table.iterator.minc.slow.opt.sleepTime=100", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t t", true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void maxrow() throws Exception {
+ exec("createtable t", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void merge() throws Exception {
+ exec("createtable t");
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f t");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(2, output.get().split("\n").length);
+ exec("getsplits -t accumulo.root", true);
+ assertEquals(1, output.get().split("\n").length);
+ exec("merge --all -t " + MetadataTable.NAME + "");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(1, output.get().split("\n").length);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void renametable() throws Exception {
+ exec("createtable aaaa");
+ exec("insert this is a value");
+ exec("renametable aaaa xyzzy");
+ exec("tables", true, "xyzzy", true);
+ exec("tables", true, "aaaa", false);
+ exec("scan -t xyzzy", true, "value", true);
+ exec("deletetable -f xyzzy", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void tables() throws Exception {
+ exec("createtable zzzz");
+ exec("createtable aaaa");
+ exec("notable");
+ String lst = exec("tables -l");
+ assertTrue(lst.indexOf("aaaa") < lst.indexOf("zzzz"));
+ lst = exec("tables -l -s");
+ assertTrue(lst.indexOf("zzzz") < lst.indexOf("aaaa"));
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(11, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void listscans() throws Exception {
+ exec("createtable t", true);
+ exec("config -t t -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t t -s table.iterator.scan.slow.opt.sleepTime=100", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Connector connector = getConnector();
+ Scanner s = connector.createScanner("t", Authorizations.EMPTY);
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> kv : s)
+ ;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
+ exec("sleep 0.1", true);
+ String scans = exec("listscans", true);
+ String lines[] = scans.split("\n");
+ String last = lines[lines.length - 1];
+ assertTrue(last.contains("RUNNING"));
+ String parts[] = last.split("\\|");
+ assertEquals(13, parts.length);
+ String hostPortPattern = ".+:\\d+";
+ String tserver = parts[0].trim();
+ assertTrue(tserver.matches(hostPortPattern));
+ assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
+ String client = parts[1].trim();
+ assertTrue(client.matches(hostPortPattern));
+ // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
+ thread.join();
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void testPertableClasspath() throws Exception {
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable ptc", true);
+ exec("config -t ptc -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t ptc -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline -w ptc");
+ exec("online -w ptc");
+
+ exec("table ptc", true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f ptc", true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void trace() throws Exception {
+ exec("trace on", true);
+ exec("createtable t", true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f t");
+ exec("sleep 1");
+ String trace = exec("trace off");
+ System.out.println(trace);
+ assertTrue(trace.contains("sendMutations"));
+ assertTrue(trace.contains("startScan"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void badLogin() throws Exception {
+ input.set(ROOT_PASSWORD + "\n");
+ String err = exec("user NoSuchUser", false);
+ assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser"));
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void namespaces() throws Exception {
+ exec("namespaces", true, "(default)", true); // default namespace for display purposes only
+ exec("namespaces", true, Namespaces.ACCUMULO_NAMESPACE, true);
+ exec("createnamespace thing1", true);
+ String namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing1"));
+
+ exec("renamenamespace thing1 thing2");
+ namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing2"));
+ assertTrue(!namespaces.contains("thing1"));
+
+ // can't delete a namespace that still contains tables, unless you do -f
+ exec("createtable thing2.thingy", true);
+ exec("deletenamespace thing2");
+ exec("y");
+ exec("namespaces", true, "thing2", true);
+
+ exec("du -ns thing2", true, "thing2.thingy", true);
+
+ // all "TableOperation" commands can take a namespace
+ exec("offline -ns thing2", true);
+ exec("online -ns thing2", true);
+ exec("flush -ns thing2", true);
+ exec("compact -ns thing2", true);
+ exec("createnamespace testers3", true);
+ exec("createtable testers3.1", true);
+ exec("createtable testers3.2", true);
+ exec("deletetable -ns testers3 -f", true);
+ exec("tables", true, "testers3.1", false);
+ exec("namespaces", true, "testers3", true);
+ exec("deletenamespace testers3 -f", true);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -ns thing2 -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("listiter -ns thing2 -scan", true, "Summing", true);
+ exec("deleteiter -ns thing2 -n name -scan", true);
+ exec("createuser dude");
+ exec("pass");
+ exec("pass");
+ exec("grant Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+ exec("revoke Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+
+ // properties override and such
+ exec("config -ns thing2 -s table.file.max=44444", true);
+ exec("config -ns thing2", true, "44444", true);
+ exec("config -t thing2.thingy", true, "44444", true);
+ exec("config -t thing2.thingy -s table.file.max=55555", true);
+ exec("config -t thing2.thingy", true, "55555", true);
+
+ // can copy properties when creating
+ exec("createnamespace thing3 -cc thing2", true);
+ exec("config -ns thing3", true, "44444", true);
+ exec("createnamespace thing4 -ctc thing2.thingy", true);
+ exec("config -ns thing4", true, "55555", true);
+
+ exec("deletenamespace -f thing2", true);
+ exec("namespaces", true, "thing2", false);
+ exec("tables", true, "thing2.thingy", false);
+
+ // put constraints on a namespace
+ exec("constraint -ns thing4 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ exec("createtable thing4.constrained", true);
+ exec("table thing4.constrained", true);
+ // should fail
+ exec("insert r cf cq abc", false);
+ exec("constraint -l", true, "NumericValueConstraint", true);
+ exec("constraint -ns thing4 -d 2");
+ exec("sleep 1");
+ exec("insert r cf cq abc", true);
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void scans() throws Exception {
+ exec("createtable t");
+ make10();
+ String result = exec("scan -np -b row1 -e row1");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row3 -e row5");
+ assertEquals(4, result.split("\n").length);
+ result = exec("scan -np -r row3");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row:");
+ assertEquals(1, result.split("\n").length);
+ result = exec("scan -np -b row");
+ assertEquals(11, result.split("\n").length);
+ result = exec("scan -np -e row:");
+ assertEquals(11, result.split("\n").length);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void whoami() throws Exception {
+ assertTrue(exec("whoami", true).contains("root"));
+ input.set("secret\nsecret\n");
+ exec("createuser test_user");
+ exec("setauths -u test_user -s 12,3,4");
+ String auths = exec("getauths -u test_user");
+ assertTrue(auths.contains("3") && auths.contains("12") && auths.contains("4"));
+ input.set("secret\n");
+ exec("user test_user", true);
+ assertTrue(exec("whoami", true).contains("test_user"));
+ input.set(ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
+
+ private int countFiles() throws IOException {
+ exec("scan -t " + MetadataTable.NAME + " -np -c file");
+ return output.get().split("\n").length - 1;
+ }
+}
[08/15] git commit: ACCUMULO-2235 Fix up formatter for madrob and
make the du test a little less sensitive to other tests
Posted by el...@apache.org.
ACCUMULO-2235 Fix up formatter for madrob and make the du test a little less sensitive to other tests
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/00bd820e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/00bd820e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/00bd820e
Branch: refs/heads/master
Commit: 00bd820ef9ff016305241504eb83e3c6e90dd873
Parents: 956c32f
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:20:46 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:20:46 2014 -0500
----------------------------------------------------------------------
.../core/util/shell/commands/DUCommand.java | 26 ++--
.../apache/accumulo/test/ShellServerTest.java | 156 ++++++++++---------
2 files changed, 93 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/00bd820e/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
index b03f516..ca80e37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
@@ -35,11 +35,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
public class DUCommand extends Command {
-
+
private Option optTablePattern, optHumanReadble;
-
+
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws IOException, TableNotFoundException {
-
+
final SortedSet<String> tablesToFlush = new TreeSet<String>(Arrays.asList(cl.getArgs()));
boolean prettyPrint = cl.hasOption(optHumanReadble.getOpt()) ? true : false;
@@ -52,12 +52,12 @@ public class DUCommand extends Command {
}
}
}
-
+
// If we didn't get any tables, and we have a table selected, add the current table
if (tablesToFlush.isEmpty() && !shellState.getTableName().isEmpty()) {
tablesToFlush.add(shellState.getTableName());
}
-
+
try {
final AccumuloConfiguration acuConf = new ConfigurationCopy(shellState.getConnector().instanceOperations().getSystemConfiguration());
TableDiskUsage.printDiskUsage(acuConf, tablesToFlush, FileSystem.get(new Configuration()), shellState.getConnector(), new Printer() {
@@ -69,40 +69,40 @@ public class DUCommand extends Command {
throw new RuntimeException(ex);
}
}
-
+
}, prettyPrint);
} catch (Exception ex) {
throw new RuntimeException(ex);
}
return 0;
}
-
+
@Override
public String description() {
return "prints how much space, in bytes, is used by files referenced by a table. When multiple tables are specified it prints how much space, in bytes, is used by files shared between tables, if any.";
}
-
+
@Override
public Options getOptions() {
final Options o = new Options();
-
+
optTablePattern = new Option("p", "pattern", true, "regex pattern of table names");
optTablePattern.setArgName("pattern");
-
+
optHumanReadble = new Option("h", "human-readable", false, "format large sizes to human readable units");
optHumanReadble.setArgName("human readable output");
o.addOption(optTablePattern);
o.addOption(optHumanReadble);
-
+
return o;
}
-
+
@Override
public String usage() {
return getName() + " <table>{ <table>}";
}
-
+
@Override
public int numArgs() {
return Shell.NO_FIXED_ARG_LENGTH_CHECK;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/00bd820e/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index bc73c9f..80e56d9 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -64,34 +64,34 @@ import org.junit.rules.TemporaryFolder;
public class ShellServerTest {
public static class TestOutputStream extends OutputStream {
StringBuilder sb = new StringBuilder();
-
+
@Override
public void write(int b) throws IOException {
sb.append((char) (0xff & b));
}
-
+
public String get() {
return sb.toString();
}
-
+
public void clear() {
sb.setLength(0);
}
}
-
+
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
public static MiniAccumuloCluster cluster;
public static TestOutputStream output;
public static Shell shell;
private static Process traceProcess;
-
+
static String exec(String cmd) throws IOException {
output.clear();
shell.execCommand(cmd, true, true);
return output.get();
}
-
+
static String exec(String cmd, boolean expectGoodExit) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
@@ -100,11 +100,11 @@ public class ShellServerTest {
assertBadExit("", true);
return result;
}
-
+
static String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
return exec(cmd, expectGoodExit, expectString, true);
}
-
+
static String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
@@ -113,15 +113,15 @@ public class ShellServerTest {
assertBadExit(expectString, stringPresent);
return result;
}
-
+
static void assertGoodExit(String s, boolean stringPresent) {
Shell.log.debug(output.get());
assertEquals(0, shell.getExitCode());
-
+
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
-
+
static void assertBadExit(String s, boolean stringPresent) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
@@ -129,16 +129,16 @@ public class ShellServerTest {
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
shell.resetExitCode();
}
-
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
folder.create();
MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
cluster = new MiniAccumuloCluster(cfg);
cluster.start();
-
+
System.setProperty("HOME", folder.getRoot().getAbsolutePath());
-
+
// start the shell
output = new TestOutputStream();
shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
@@ -147,23 +147,23 @@ public class ShellServerTest {
exec("quit", true);
shell.start();
shell.setExit(false);
-
+
// use reflection to call this method so it does not need to be made public
Method method = cluster.getClass().getDeclaredMethod("exec", Class.class, String[].class);
method.setAccessible(true);
traceProcess = (Process) method.invoke(cluster, TraceServer.class, new String[0]);
-
+
// give the tracer some time to start
UtilWaitThread.sleep(1000);
}
-
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
cluster.stop();
traceProcess.destroy();
folder.delete();
}
-
+
@Test(timeout = 30000)
public void exporttableImporttable() throws Exception {
// exporttable / importtable
@@ -185,7 +185,7 @@ public class ShellServerTest {
exec("deletetable -f t", true);
exec("deletetable -f t2", true);
}
-
+
private DistCp newDistCp() {
try {
@SuppressWarnings("unchecked")
@@ -205,7 +205,7 @@ public class ShellServerTest {
}
throw new RuntimeException("Unexpected constructors for DistCp");
}
-
+
@Test(timeout = 30000)
public void setscaniterDeletescaniter() throws Exception {
// setscaniter, deletescaniter
@@ -219,9 +219,9 @@ public class ShellServerTest {
exec("deletescaniter -n name", true);
exec("scan", true, "1", true);
exec("deletetable -f t");
-
+
}
-
+
@Test(timeout = 30000)
public void execfile() throws Exception {
// execfile
@@ -230,9 +230,9 @@ public class ShellServerTest {
writer.println("about");
writer.close();
exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
-
+
}
-
+
@Test(timeout = 30000)
public void egrep() throws Exception {
// egrep
@@ -242,13 +242,17 @@ public class ShellServerTest {
assertTrue(lines.split("\n").length - 1 == 3);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void du() throws Exception {
+ // create and delete a table so we get out of a table context in the shell
+ exec("createtable du_test_table", true);
+ exec("deletetable -f du_test_table", true);
+
// Calling du not in a table context shouldn't throw an error
output.clear();
exec("du", true, "", true);
-
+
output.clear();
exec("createtable t");
make10();
@@ -258,10 +262,10 @@ public class ShellServerTest {
shell.execCommand("du -h", false, false);
String o = output.get();
// for some reason, there's a bit of fluctuation
- assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[t\\]\\n"));
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*2[6-7][0-9]B\\s\\[t\\]\\n"));
exec("deletetable -f t");
}
-
+
@Test(timeout = 1000)
public void debug() throws Exception {
exec("debug", true, "off", true);
@@ -272,7 +276,7 @@ public class ShellServerTest {
exec("debug debug", false);
exec("debug debug debug", false);
}
-
+
@Test(timeout = 30000)
public void user() throws Exception {
// createuser, deleteuser, user, users, droptable, grant, revoke
@@ -306,7 +310,7 @@ public class ShellServerTest {
exec("deleteuser xyzzy", true);
exec("users", true, "xyzzy", false);
}
-
+
@Test(timeout = 30000)
public void iter() throws Exception {
// setshelliter, listshelliter, deleteshelliter
@@ -347,9 +351,9 @@ public class ShellServerTest {
exec("listiter -scan", true, "Iterator name", false);
exec("listiter -scan", true, "Iterator xyzzy", true);
exec("deletetable -f t");
-
+
}
-
+
@Test(timeout = 30000)
public void notable() throws Exception {
// notable
@@ -361,7 +365,7 @@ public class ShellServerTest {
assertFalse(output.get().contains(" xyzzy>"));
exec("deletetable -f xyzzy");
}
-
+
@Test(timeout = 30000)
public void sleep() throws Exception {
// sleep
@@ -371,7 +375,7 @@ public class ShellServerTest {
assertTrue(diff >= 200);
assertTrue(diff < 400);
}
-
+
@Test(timeout = 30000)
public void addauths() throws Exception {
// addauths
@@ -385,7 +389,7 @@ public class ShellServerTest {
exec("scan -s bar", true, "[foo]", false);
exec("deletetable -f xyzzy");
}
-
+
@Test(timeout = 30000)
public void byeQuitExit() throws Exception {
// bye, quit, exit
@@ -396,20 +400,20 @@ public class ShellServerTest {
shell.setExit(false);
}
}
-
+
@Test(timeout = 30000)
public void classpath() throws Exception {
// classpath
exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
}
-
+
@Test(timeout = 30000)
public void clearCls() throws Exception {
// clear/cls
exec("cls", true, "[1;1H");
exec("clear", true, "[2J");
}
-
+
@Test(timeout = 30000)
public void clonetable() throws Exception {
// clonetable
@@ -454,7 +458,7 @@ public class ShellServerTest {
exec("deletetable -f clone");
exec("deletetable -f c");
}
-
+
@Test(timeout = 30000)
public void constraint() throws Exception {
// constraint
@@ -467,7 +471,7 @@ public class ShellServerTest {
exec("constraint -l -t c", true, "VisibilityConstraint=1", false);
exec("deletetable -f c");
}
-
+
@Test(timeout = 30000)
public void deletemany() throws Exception {
// deletemany
@@ -494,7 +498,7 @@ public class ShellServerTest {
assertEquals(9, countkeys("t"));
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void deleterows() throws Exception {
// deleterows
@@ -508,7 +512,7 @@ public class ShellServerTest {
assertTrue(base + 2 == countFiles());
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void groups() throws Exception {
exec("createtable t");
@@ -517,7 +521,7 @@ public class ShellServerTest {
exec("getgroups -t t", true, "num=1,2,3", true);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void grep() throws Exception {
exec("createtable t", true);
@@ -526,7 +530,7 @@ public class ShellServerTest {
exec("grep row5", true, "row5", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void help() throws Exception {
exec("help -np", true, "Help Commands", true);
@@ -540,7 +544,7 @@ public class ShellServerTest {
exec("help " + c, true);
}
}
-
+
// @Test(timeout = 30000)
public void history() throws Exception {
exec("history -c", true);
@@ -549,7 +553,7 @@ public class ShellServerTest {
exec("history", true, "unusualstring", true);
exec("history", true, "history", true);
}
-
+
@Test(timeout = 30000)
public void importDirectory() throws Exception {
Configuration conf = new Configuration();
@@ -582,12 +586,12 @@ public class ShellServerTest {
exec("scan -r 00000099", true, "00000099", true);
exec("deletetable -f t");
}
-
+
@Test(timeout = 30000)
public void info() throws Exception {
exec("info", true, Constants.VERSION, true);
}
-
+
@Test(timeout = 30000)
public void interpreter() throws Exception {
exec("createtable t", true);
@@ -599,7 +603,7 @@ public class ShellServerTest {
exec("scan -b 02", true, "value", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void listcompactions() throws Exception {
exec("createtable t", true);
@@ -618,7 +622,7 @@ public class ShellServerTest {
assertEquals(12, parts.length);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void maxrow() throws Exception {
exec("createtable t", true);
@@ -631,7 +635,7 @@ public class ShellServerTest {
exec("maxrow", true, "ccc", true);
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void merge() throws Exception {
exec("createtable t");
@@ -646,7 +650,7 @@ public class ShellServerTest {
exec("getsplits -t !METADATA", true);
assertEquals(2, output.get().split("\n").length);
}
-
+
@Test(timeout = 30000)
public void ping() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -655,11 +659,11 @@ public class ShellServerTest {
if (output.get().split("\n").length == 3)
break;
UtilWaitThread.sleep(1000);
-
+
}
assertEquals(3, output.get().split("\n").length);
}
-
+
@Test(timeout = 30000)
public void renametable() throws Exception {
exec("createtable aaaa");
@@ -670,7 +674,7 @@ public class ShellServerTest {
exec("scan -t xyzzy", true, "value", true);
exec("deletetable -f xyzzy", true);
}
-
+
@Test(timeout = 30000)
public void systempermission() throws Exception {
exec("systempermissions");
@@ -678,7 +682,7 @@ public class ShellServerTest {
exec("tablepermissions", true);
assertEquals(6, output.get().split("\n").length - 1);
}
-
+
@Test(timeout = 30000)
public void listscans() throws Exception {
exec("createtable t", true);
@@ -713,52 +717,52 @@ public class ShellServerTest {
thread.join();
exec("deletetable -f t", true);
}
-
+
@Test(timeout = 30000)
public void testPertableClasspath() throws Exception {
File fooFilterJar = File.createTempFile("FooFilter", ".jar");
FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
fooFilterJar.deleteOnExit();
-
+
File fooConstraintJar = File.createTempFile("FooConstraint", ".jar");
FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
fooConstraintJar.deleteOnExit();
-
+
exec(
"config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
true);
-
+
exec("createtable ptc", true);
exec("config -t ptc -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
-
+
UtilWaitThread.sleep(200);
-
- // We can't use the setiter command as Filter implements OptionDescriber which
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
// forces us to enter more input that I don't know how to input
// Instead, we can just manually set the property on the table.
exec("config -t ptc -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
-
+
exec("insert foo f q v", true);
-
+
UtilWaitThread.sleep(100);
-
+
exec("scan -np", true, "foo", false);
-
+
exec("constraint -a FooConstraint", true);
-
+
exec("offline ptc");
UtilWaitThread.sleep(500);
exec("online ptc");
-
+
exec("table ptc", true);
exec("insert foo f q v", false);
exec("insert ok foo q v", true);
-
+
exec("deletetable -f ptc", true);
exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
-
+
}
-
+
@Test(timeout = 30000)
public void trace() throws Exception {
exec("trace on", true);
@@ -772,21 +776,21 @@ public class ShellServerTest {
assertTrue(trace.contains("update"));
assertTrue(trace.contains("DeleteTable"));
}
-
+
private int countkeys(String table) throws IOException {
exec("scan -np -t " + table);
return output.get().split("\n").length - 1;
}
-
+
private void make10() throws IOException {
for (int i = 0; i < 10; i++) {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
-
+
private int countFiles() throws IOException {
exec("scan -t !METADATA -np -c file");
return output.get().split("\n").length - 1;
}
-
+
}
[02/15] git commit: ACCUMULO-2183 Look for the thrift
SecurityErrorCode since this is coming from the ThriftSecurityException.
Posted by el...@apache.org.
ACCUMULO-2183 Look for the thrift SecurityErrorCode since this is coming from the ThriftSecurityException.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/956c32f8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/956c32f8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/956c32f8
Branch: refs/heads/master
Commit: 956c32f8fa258ea0b87f267675cdcec91e16dce1
Parents: fbed7af
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 12:15:10 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 12:15:10 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/test/randomwalk/security/Validate.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/956c32f8/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
index 2a3e445..047bba3 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/security/Validate.java
@@ -108,7 +108,7 @@ public class Validate extends Test {
auths = WalkingSecurity.get(state).getUserAuthorizations(WalkingSecurity.get(state).getTabCredentials());
accuAuths = conn.securityOperations().getUserAuthorizations(WalkingSecurity.get(state).getTabUserName());
} catch (ThriftSecurityException ae) {
- if (ae.getCode().equals(SecurityErrorCode.USER_DOESNT_EXIST)) {
+ if (ae.getCode().equals(org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode.USER_DOESNT_EXIST)) {
if (tableUserExists)
throw new AccumuloException("Table user didn't exist when they should.", ae);
else
[13/15] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9440d15c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9440d15c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9440d15c
Branch: refs/heads/master
Commit: 9440d15cbe0a1639c509bb37d2e30830efdfddc9
Parents: ecdd852 a147acd
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 13:05:00 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 13:05:00 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/RangeInputSplit.java | 2 +-
.../core/util/shell/commands/DUCommand.java | 4 +-
.../core/client/mapreduce/InputFormatBase.java | 1634 ++++++++++++++++++
.../test/continuous/ContinuousVerify.java | 80 +-
.../org/apache/accumulo/test/ShellServerIT.java | 112 +-
5 files changed, 1734 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 85a0104,f7b2263..831d10b
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@@ -369,8 -324,8 +369,8 @@@ public class RangeInputSplit extends In
if (null == zooKeepers) {
return null;
}
-
+
- ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+ ZooKeeperInstance zki = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(getInstanceName()).withZkHosts(getZooKeepers()));
// Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
// when it's present
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
index a0fcc10,ca80e37..660ec6c
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DUCommand.java
@@@ -67,18 -52,25 +67,18 @@@ public class DUCommand extends Command
}
}
}
-
+
// If we didn't get any tables, and we have a table selected, add the current table
- if (tablesToFlush.isEmpty() && !shellState.getTableName().isEmpty()) {
- tablesToFlush.add(shellState.getTableName());
+ if (tables.isEmpty() && !shellState.getTableName().isEmpty()) {
+ tables.add(shellState.getTableName());
}
-
+
try {
- final AccumuloConfiguration acuConf = new ConfigurationCopy(shellState.getConnector().instanceOperations().getSystemConfiguration());
- TableDiskUsage.printDiskUsage(acuConf, tablesToFlush, FileSystem.get(new Configuration()), shellState.getConnector(), new Printer() {
- @Override
- public void print(String line) {
- try {
- shellState.getReader().printString(line + "\n");
- } catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- }
-
- }, prettyPrint);
+ String valueFormat = prettyPrint ? "%9s" : "%,24d";
+ for (DiskUsage usage : shellState.getConnector().tableOperations().getDiskUsage(tables)) {
+ Object value = prettyPrint ? NumUtil.bigNumberForSize(usage.getUsage()) : usage.getUsage();
+ shellState.getReader().println(String.format(valueFormat + " %s", value, usage.getTables()));
+ }
} catch (Exception ex) {
throw new RuntimeException(ex);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 09ebdd1,a9d7153..fbcb6fa
--- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@@ -188,10 -188,9 +188,10 @@@ public class ContinuousVerify extends C
Opts opts = new Opts();
opts.parseArgs(this.getClass().getName(), args);
+ @SuppressWarnings("deprecation")
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
-
+
job.setInputFormatClass(AccumuloInputFormat.class);
opts.setAccumuloConfigs(job);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9440d15c/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index 2a06143,0000000..81f9ef2
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@@ -1,1048 -1,0 +1,1050 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.util.Map.Entry;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ShellServerIT extends SimpleMacIT {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ public static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() throws IOException {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+
+ public static TestOutputStream output;
+ public static StringInputStream input;
+ public static Shell shell;
+ private static Process traceProcess;
+
+ static String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ static String exec(String cmd, boolean expectGoodExit) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit("", true);
+ else
+ assertBadExit("", true);
+ return result;
+ }
+
+ static String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true);
+ }
+
+ static String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
+ assertGoodExit(expectString, stringPresent);
+ else
+ assertBadExit(expectString, stringPresent);
+ return result;
+ }
+
+ static void assertGoodExit(String s, boolean stringPresent) {
+ Shell.log.info(output.get());
+ assertEquals(0, shell.getExitCode());
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
+ static void assertBadExit(String s, boolean stringPresent) {
+ Shell.log.debug(output.get());
+ assertTrue(shell.getExitCode() > 0);
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // history file is updated in $HOME
+ System.setProperty("HOME", getFolder().getAbsolutePath());
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
+ shell = new Shell(new ConsoleReader(input, output));
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(),
+ "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+
+ traceProcess = getStaticCluster().exec(TraceServer.class);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ traceProcess.destroy();
+ }
+
+ @After
+ public void deleteTables() throws Exception {
+ Connector c = getConnector();
+ for (String table : c.tableOperations().list()) {
+ if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace"))
+ try {
+ c.tableOperations().delete(table);
+ } catch (TableNotFoundException e) {
+ // don't care
+ }
+ }
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void exporttableImporttable() throws Exception {
+ // exporttable / importtable
+ exec("createtable t -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t t -s table.split.threshold=345M", true);
+ exec("offline t", true);
+ String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString();
+ exec("exporttable -t t " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable t2 " + import_, true);
+ exec("config -t t2 -np", true, "345M", true);
+ exec("getsplits -t t2", true, "row5", true);
+ exec("constraint --list -t t2", true, "VisibilityConstraint=2", true);
+ exec("onlinetable t", true);
+ exec("deletetable -f t", true);
+ exec("deletetable -f t2", true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void setscaniterDeletescaniter() throws Exception {
+ // setscaniter, deletescaniter
+ exec("createtable t");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING");
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f t");
+
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder());
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void egrep() throws Exception {
+ // egrep
+ exec("createtable t");
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void du() throws Exception {
++ // create and delete a table so we get out of a table context in the shell
++ exec("createtable du_test_table", true);
++ exec("deletetable -f du_test_table", true);
++
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
-
++
+ output.clear();
+ exec("createtable t");
+ make10();
+ exec("flush -t t -w");
+ exec("du t", true, " [t]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*[1-9][0-9][0-9]\\s\\[t\\]\\n"));
+ exec("deletetable -f t");
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void user() throws Exception {
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ input.set("secret\nsecret\n");
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (" + MetadataTable.NAME + "): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ input.set("secret\nsecret\n");
+ exec("user xyzzy", true);
+ exec("createtable t", true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f t", true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void iter() throws Exception {
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable t");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n name", false);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f t");
+ // list iter
+ exec("createtable t");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n name", false);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f t");
+
+ }
-
- @Test(timeout = 30 * 1000)
++
++ @Test(timeout = 30000)
+ public void setIterOptionPrompt() throws Exception {
+ Connector conn = getConnector();
+ String tableName = "setIterOptionPrompt";
-
++
+ exec("createtable " + tableName);
+ input.set("\n\n");
+ // Setting a non-optiondescriber with no name should fail
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", false);
-
++
+ // Name as option will work
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+
+ String expectedKey = "table.iterator.scan.cfcounter";
+ String expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ TableOperations tops = conn.tableOperations();
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
-
++
+ exec("createtable " + tableName, true);
-
++
+ input.set("customcfcounter\n\n");
-
++
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
-
++
+ exec("createtable " + tableName, true);
-
++
+ input.set("customcfcounter\nname1 value1\nname2 value2\n\n");
-
++
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name1";
+ expectedValue = "value1";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
-
++
+ exec("createtable " + tableName, true);
-
++
+ input.set("\nname1 value1.1,value1.2,value1.3\nname2 value2\n\n");
-
++
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+ expectedKey = "table.iterator.scan.cfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name1";
+ expectedValue = "value1.1,value1.2,value1.3";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ }
+
+ protected void checkTableForProperty(TableOperations tops, String tableName, String expectedKey, String expectedValue) throws Exception {
+ for (int i = 0; i < 5; i++) {
+ for (Entry<String,String> entry : tops.getProperties(tableName)) {
+ if (expectedKey.equals(entry.getKey())) {
+ assertEquals(expectedValue, entry.getValue());
+ return;
+ }
+ }
+ Thread.sleep(500);
+ }
+
+ fail("Failed to find expected property on " + tableName + ": " + expectedKey + "=" + expectedValue);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void notable() throws Exception {
+ // notable
+ exec("createtable xyzzy", true);
+ exec("scan", true, " xyzzy>", true);
+ assertTrue(output.get().contains(" xyzzy>"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" xyzzy>"));
+ exec("deletetable -f xyzzy");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue(diff >= 200);
+ assertTrue(diff < 400);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void addauths() throws Exception {
+ // addauths
+ exec("createtable xyzzy -evc");
+ exec("insert a b c d -l foo", false, "does not have authorization", true);
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f xyzzy");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ if (shell.getReader().getTerminal().isAnsiSupported()) {
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ } else {
+ exec("cls", false, "does not support");
+ exec("clear", false, "does not support");
+ }
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void testCompactions() throws IOException {
+ // compact
+ exec("createtable c");
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles();
+ // merge two files into one
+ exec("compact -t c -w");
+ assertTrue(countFiles() < oldCount);
+ exec("addsplits -t c f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
+ oldCount = countFiles();
+ // at this point there are 3 files in the default tablet
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertTrue(countFiles() == oldCount - 2);
+ exec("compact -w");
+ assertTrue(countFiles() == oldCount - 2);
+ exec("merge --all -t c");
+ exec("compact -w");
+ assertTrue(countFiles() == oldCount - 3);
+ exec("deletetable -f c");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void clonetable() throws Exception {
+ // clonetable
+ exec("createtable orig -evc");
+ exec("config -t orig -s table.split.threshold=123M", true);
+ exec("addsplits -t orig a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable orig clone");
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t clone", true, "VisibilityConstraint=2", true);
+ exec("config -t clone -np", true, "123M", true);
+ exec("getsplits -t clone", true, "a\nb\nc\n");
+ exec("deletetable -f orig");
+ exec("deletetable -f clone");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void constraint() throws Exception {
+ // constraint
+ exec("constraint -l -t " + MetadataTable.NAME + "", true, "MetadataConstraints=1", true);
+ exec("createtable c -evc");
+ exec("constraint -l -t c", true, "VisibilityConstraint=2", true);
+ exec("constraint -t c -d 2", true, "Removed constraint 2 from table c");
+ // wait for zookeeper updates to propagate
+ UtilWaitThread.sleep(1000);
+ exec("constraint -l -t c", true, "VisibilityConstraint=2", false);
+ exec("deletetable -f c");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void deletemany() throws Exception {
+ // deletemany
+ exec("createtable t");
+ make10();
+ assertEquals(10, countkeys("t"));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys("t"));
+ exec("scan -t t -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys("t"));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys("t"));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys("t"));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys("t"));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys("t"));
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void deleterows() throws Exception {
+ // deleterows
+ int base = countFiles();
+ exec("createtable t");
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t t");
+ assertEquals(base + 3, countFiles());
+ exec("deleterows -t t -b row5 -e row7", true);
+ assertEquals(base + 2, countFiles());
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void groups() throws Exception {
+ exec("createtable t");
+ exec("setgroups -t t alpha=a,b,c num=3,2,1");
+ exec("getgroups -t t", true, "alpha=a,b,c", true);
+ exec("getgroups -t t", true, "num=1,2,3", true);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void grep() throws Exception {
+ exec("createtable t", true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f t", true);
+ }
+
- @Test
- // (timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
-
- // @Test(timeout = 30 * 1000)
++ // @Test(timeout = 30000)
+ public void history() throws Exception {
+ exec("history -c", true);
+ exec("createtable unusualstring");
+ exec("deletetable -f unusualstring");
+ exec("history", true, "unusualstring", true);
+ exec("history", true, "history", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void importDirectory() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = new File(getFolder(), "import");
+ importDir.mkdir();
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = new File(getFolder(), "errors");
+ errorsDir.mkdir();
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ assertEquals(0, shell.getExitCode());
+ exec("createtable t", true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void interpreter() throws Exception {
+ exec("createtable t", true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+ UtilWaitThread.sleep(500);
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void listcompactions() throws Exception {
+ exec("createtable t", true);
+ exec("config -t t -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t t -s table.iterator.minc.slow.opt.sleepTime=100", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t t", true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void maxrow() throws Exception {
+ exec("createtable t", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void merge() throws Exception {
+ exec("createtable t");
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f t");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(2, output.get().split("\n").length);
+ exec("getsplits -t accumulo.root", true);
+ assertEquals(1, output.get().split("\n").length);
+ exec("merge --all -t " + MetadataTable.NAME + "");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(1, output.get().split("\n").length);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void renametable() throws Exception {
+ exec("createtable aaaa");
+ exec("insert this is a value");
+ exec("renametable aaaa xyzzy");
+ exec("tables", true, "xyzzy", true);
+ exec("tables", true, "aaaa", false);
+ exec("scan -t xyzzy", true, "value", true);
+ exec("deletetable -f xyzzy", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void tables() throws Exception {
+ exec("createtable zzzz");
+ exec("createtable aaaa");
+ exec("notable");
+ String lst = exec("tables -l");
+ assertTrue(lst.indexOf("aaaa") < lst.indexOf("zzzz"));
+ lst = exec("tables -l -s");
+ assertTrue(lst.indexOf("zzzz") < lst.indexOf("aaaa"));
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(11, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void listscans() throws Exception {
+ exec("createtable t", true);
+ exec("config -t t -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t t -s table.iterator.scan.slow.opt.sleepTime=100", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Connector connector = getConnector();
+ Scanner s = connector.createScanner("t", Authorizations.EMPTY);
+ for (@SuppressWarnings("unused")
+ Entry<Key,Value> kv : s)
+ ;
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
+ exec("sleep 0.1", true);
+ String scans = exec("listscans", true);
+ String lines[] = scans.split("\n");
+ String last = lines[lines.length - 1];
+ assertTrue(last.contains("RUNNING"));
+ String parts[] = last.split("\\|");
+ assertEquals(13, parts.length);
+ String hostPortPattern = ".+:\\d+";
+ String tserver = parts[0].trim();
+ assertTrue(tserver.matches(hostPortPattern));
+ assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
+ String client = parts[1].trim();
+ assertTrue(client.matches(hostPortPattern));
+ // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
+ thread.join();
+ exec("deletetable -f t", true);
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void testPertableClasspath() throws Exception {
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable ptc", true);
+ exec("config -t ptc -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t ptc -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline -w ptc");
+ exec("online -w ptc");
+
+ exec("table ptc", true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f ptc", true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void trace() throws Exception {
+ exec("trace on", true);
+ exec("createtable t", true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f t");
+ exec("sleep 1");
+ String trace = exec("trace off");
+ System.out.println(trace);
+ assertTrue(trace.contains("sendMutations"));
+ assertTrue(trace.contains("startScan"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void badLogin() throws Exception {
+ input.set(ROOT_PASSWORD + "\n");
+ String err = exec("user NoSuchUser", false);
+ assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser"));
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void namespaces() throws Exception {
+ exec("namespaces", true, "(default)", true); // default namespace for display purposes only
+ exec("namespaces", true, Namespaces.ACCUMULO_NAMESPACE, true);
+ exec("createnamespace thing1", true);
+ String namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing1"));
+
+ exec("renamenamespace thing1 thing2");
+ namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing2"));
+ assertTrue(!namespaces.contains("thing1"));
+
+ // can't delete a namespace that still contains tables, unless you do -f
+ exec("createtable thing2.thingy", true);
+ exec("deletenamespace thing2");
+ exec("y");
+ exec("namespaces", true, "thing2", true);
+
+ exec("du -ns thing2", true, "thing2.thingy", true);
+
+ // all "TableOperation" commands can take a namespace
+ exec("offline -ns thing2", true);
+ exec("online -ns thing2", true);
+ exec("flush -ns thing2", true);
+ exec("compact -ns thing2", true);
+ exec("createnamespace testers3", true);
+ exec("createtable testers3.1", true);
+ exec("createtable testers3.2", true);
+ exec("deletetable -ns testers3 -f", true);
+ exec("tables", true, "testers3.1", false);
+ exec("namespaces", true, "testers3", true);
+ exec("deletenamespace testers3 -f", true);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -ns thing2 -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("listiter -ns thing2 -scan", true, "Summing", true);
+ exec("deleteiter -ns thing2 -n name -scan", true);
+ exec("createuser dude");
+ exec("pass");
+ exec("pass");
+ exec("grant Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+ exec("revoke Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+
+ // properties override and such
+ exec("config -ns thing2 -s table.file.max=44444", true);
+ exec("config -ns thing2", true, "44444", true);
+ exec("config -t thing2.thingy", true, "44444", true);
+ exec("config -t thing2.thingy -s table.file.max=55555", true);
+ exec("config -t thing2.thingy", true, "55555", true);
+
+ // can copy properties when creating
+ exec("createnamespace thing3 -cc thing2", true);
+ exec("config -ns thing3", true, "44444", true);
+ exec("createnamespace thing4 -ctc thing2.thingy", true);
+ exec("config -ns thing4", true, "55555", true);
+
+ exec("deletenamespace -f thing2", true);
+ exec("namespaces", true, "thing2", false);
+ exec("tables", true, "thing2.thingy", false);
+
+ // put constraints on a namespace
+ exec("constraint -ns thing4 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ exec("createtable thing4.constrained", true);
+ exec("table thing4.constrained", true);
+ // should fail
+ exec("insert r cf cq abc", false);
+ exec("constraint -l", true, "NumericValueConstraint", true);
+ exec("constraint -ns thing4 -d 2");
+ exec("sleep 1");
+ exec("insert r cf cq abc", true);
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void scans() throws Exception {
+ exec("createtable t");
+ make10();
+ String result = exec("scan -np -b row1 -e row1");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row3 -e row5");
+ assertEquals(4, result.split("\n").length);
+ result = exec("scan -np -r row3");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row:");
+ assertEquals(1, result.split("\n").length);
+ result = exec("scan -np -b row");
+ assertEquals(11, result.split("\n").length);
+ result = exec("scan -np -e row:");
+ assertEquals(11, result.split("\n").length);
+ exec("deletetable -f t");
+ }
+
- @Test(timeout = 30 * 1000)
++ @Test(timeout = 30000)
+ public void whoami() throws Exception {
+ assertTrue(exec("whoami", true).contains("root"));
+ input.set("secret\nsecret\n");
+ exec("createuser test_user");
+ exec("setauths -u test_user -s 12,3,4");
+ String auths = exec("getauths -u test_user");
+ assertTrue(auths.contains("3") && auths.contains("12") && auths.contains("4"));
+ input.set("secret\n");
+ exec("user test_user", true);
+ assertTrue(exec("whoami", true).contains("test_user"));
+ input.set(ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
+
+ private int countFiles() throws IOException {
+ exec("scan -t " + MetadataTable.NAME + " -np -c file");
+ return output.get().split("\n").length - 1;
+ }
+}