You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/03/11 19:25:10 UTC
[01/23] git commit: ACCUMULO-2438 Rename the test so that surefire
doesn't pick it up as a unit test
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-2061 43a7ebda2 -> 2da21b6a4
ACCUMULO-2438 Rename the test so that surefire doesn't pick it up as a unit test
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4fabfbaa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4fabfbaa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4fabfbaa
Branch: refs/heads/ACCUMULO-2061
Commit: 4fabfbaa12ec71e4495af5162717677aee3a9125
Parents: 2784383
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 7 13:23:19 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 7 13:23:19 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/test/DeleteRowsIT.java | 42 ++++++++++++++++++++
.../apache/accumulo/test/TestDeleteRowsIT.java | 42 --------------------
2 files changed, 42 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4fabfbaa/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java b/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
new file mode 100644
index 0000000..190f693
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/DeleteRowsIT.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.junit.Test;
+
+public class DeleteRowsIT extends SimpleMacIT {
+
+ @Test(timeout = 60 * 1000)
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String[] tableNames = this.getTableNames(20);
+ for (String tableName : tableNames) {
+ c.tableOperations().create(tableName);
+ c.tableOperations().deleteRows(tableName, null, null);
+ Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+ assertEquals(0, FunctionalTestUtils.count(scanner));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4fabfbaa/test/src/test/java/org/apache/accumulo/test/TestDeleteRowsIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/TestDeleteRowsIT.java b/test/src/test/java/org/apache/accumulo/test/TestDeleteRowsIT.java
deleted file mode 100644
index f33a617..0000000
--- a/test/src/test/java/org/apache/accumulo/test/TestDeleteRowsIT.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.test.functional.FunctionalTestUtils;
-import org.apache.accumulo.test.functional.SimpleMacIT;
-import org.junit.Test;
-
-public class TestDeleteRowsIT extends SimpleMacIT {
-
- @Test(timeout = 60 * 1000)
- public void test() throws Exception {
- Connector c = getConnector();
- String[] tableNames = this.getTableNames(20);
- for (String tableName : tableNames) {
- c.tableOperations().create(tableName);
- c.tableOperations().deleteRows(tableName, null, null);
- Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
- assertEquals(0, FunctionalTestUtils.count(scanner));
- }
- }
-
-}
[14/23] git commit: ACCUMULO-1936 Make testInstanceOperations a more
reliable test.
Posted by el...@apache.org.
ACCUMULO-1936 Make testInstanceOperations a more reliable test.
Avoid pulling scans for users other than root and compactions for
compactions over tables other than the one we're looking at. Slow down
the slow iterator even more.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/72395e01
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/72395e01
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/72395e01
Branch: refs/heads/ACCUMULO-2061
Commit: 72395e010d3f495507d03664057a63c3d690a206
Parents: e13c273
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:26:19 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/proxy/SimpleTest.java | 110 +++++++++++++------
1 file changed, 75 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/72395e01/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
----------------------------------------------------------------------
diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
index 5573d20..77e7b9e 100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.server.TServer;
@@ -102,7 +103,8 @@ import org.junit.rules.TemporaryFolder;
* Call every method on the proxy and try to verify that it works.
*/
public class SimpleTest {
-
+ private static final Logger log = Logger.getLogger(SimpleTest.class);
+
public static TemporaryFolder folder = new TemporaryFolder();
public static final String TABLE_TEST = "test";
@@ -704,7 +706,7 @@ public class SimpleTest {
assertEquals(0, entries.results.size());
}
- @Test(timeout = 10000)
+ @Test(timeout = 60000)
public void testInstanceOperations() throws Exception {
int tservers = 0;
for (String tserver : client.getTabletServers(creds)) {
@@ -745,12 +747,14 @@ public class SimpleTest {
// create a table that's very slow, so we can look for scans/compactions
client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ // Should take 5 seconds to read every record
+ for (int i = 0; i < 20; i++) {
+ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
+ }
+
+ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
// scan
Thread t = new Thread() {
@@ -768,28 +772,45 @@ public class SimpleTest {
}
};
t.start();
+
// look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
+ List<ActiveScan> scans = new ArrayList<ActiveScan>();
+ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
+ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
+ for (ActiveScan scan : scansForServer) {
+ if ("root".equals(scan.getUser())) {
+ scans.add(scan);
+ }
+ }
+
if (!scans.isEmpty())
- break loop;
+ break;
UtilWaitThread.sleep(10);
}
}
t.join();
- assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
+ assertFalse(scans.isEmpty());
+ boolean found = false;
+ Map<String,String> map = null;
+ for (int i = 0; i < scans.size() && !found; i++) {
+ ActiveScan scan = scans.get(i);
+ if ("root".equals(scan.getUser())) {
+ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
+ assertEquals(ScanType.SINGLE, scan.getType());
+ assertEquals("slow", scan.getTable());
+
+ map = client.tableIdMap(creds);
+ assertEquals(map.get("slow"), scan.getExtent().tableId);
+ assertTrue(scan.getExtent().endRow == null);
+ assertTrue(scan.getExtent().prevEndRow == null);
+ found = true;
+ }
+ }
+
+ assertTrue("Could not find a scan against the 'slow' table", found);
+
// start a compaction
t = new Thread() {
@Override
@@ -803,27 +824,46 @@ public class SimpleTest {
}
};
t.start();
-
+
+ final String desiredTableId = map.get("slow");
+
// try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
+ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
+ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
+ // Iterate over the tservers
for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
+ // And get the compactions on each
+ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
+ for (ActiveCompaction compact : compactionsOnServer) {
+ // There might be other compactions occurring (e.g. on METADATA) in which
+ // case we want to prune out those that aren't for our slow table
+ if (desiredTableId.equals(compact.getExtent().tableId)) {
+ compactions.add(compact);
+ }
+ }
+
+ // If we found a compaction for the table we wanted, so we can stop looking
if (!compactions.isEmpty())
- break loop2;
+ break;
}
UtilWaitThread.sleep(10);
}
t.join();
+
// verify the compaction information
assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
+ for (ActiveCompaction c : compactions) {
+ if (desiredTableId.equals(c.getExtent().tableId)) {
+ assertTrue(c.inputFiles.isEmpty());
+ assertEquals(CompactionType.MINOR, c.getType());
+ assertEquals(CompactionReason.USER, c.getReason());
+ assertEquals("", c.localityGroup);
+ assertTrue(c.outputFile.contains("default_tablet"));
+
+ return;
+ }
+ }
+ fail("Expection to find running compaction for table 'slow' but did not find one");
}
@Test
@@ -1178,5 +1218,5 @@ public class SimpleTest {
accumulo.stop();
folder.delete();
}
-
-}
+
+}
\ No newline at end of file
[13/23] git commit: ACCUMULO-2447 Always set PrintWriter on Shell and
don't print to stderr.
Posted by el...@apache.org.
ACCUMULO-2447 Always set PrintWriter on Shell and don't print to stderr.
While we don't have access to the printwriter/outputstream from the jline
ConsoleReader, we can force the user to pass in one (or just make one for
stdout). This will allow unit tests to correctly squash all output into
a bytearray (or similar) instead of having to redirect stderr too (which
would have other repercussions).
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e13c2733
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e13c2733
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e13c2733
Branch: refs/heads/ACCUMULO-2061
Commit: e13c27331a42d55408427f18d97b285372f92a47
Parents: bc9cee0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:18:23 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 14:27:54 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +++++++-------------
.../accumulo/core/util/shell/ShellTest.java | 4 +++-
.../apache/accumulo/test/ShellServerTest.java | 3 ++-
3 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 656f2ae..6943b37 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@ -202,16 +202,14 @@ public class Shell extends ShellOptions {
private boolean masking = false;
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
@@ -943,13 +941,9 @@ public class Shell extends ShellOptions {
}
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
public int getExitCode() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 02e9543..8505370 100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@ -24,6 +24,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
import jline.ConsoleReader;
@@ -81,7 +82,8 @@ public class ShellTest {
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e13c2733/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index 28a4dd2..e2ebbcf 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -163,7 +163,8 @@ public class ShellServerTest {
public void setupShell() throws Exception {
// start the shell
output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)));
+ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
+ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
shell.setLogErrorsToConsole();
shell.config("-u", "root", "-p", secret, "-z", cluster.getInstanceName(), cluster.getZooKeepers());
exec("quit", true);
[02/23] git commit: ACCUMULO-2061 Fix up ServerConstants and make
ChangeSecret reset the instanceId on all Volumes
Posted by el...@apache.org.
ACCUMULO-2061 Fix up ServerConstants and make ChangeSecret reset the instanceId on all Volumes
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0c841568
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0c841568
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0c841568
Branch: refs/heads/ACCUMULO-2061
Commit: 0c841568ee715951d3ebc889c105d55328e8144b
Parents: 43a7ebd
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 7 16:35:43 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 7 16:37:33 2014 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/core/volume/Volume.java | 7 +++++++
.../apache/accumulo/core/volume/VolumeImpl.java | 17 +++++------------
.../java/org/apache/accumulo/server/Accumulo.java | 11 +++++++----
.../apache/accumulo/server/ServerConstants.java | 9 +++++----
.../apache/accumulo/server/fs/VolumeManager.java | 6 ++++++
.../accumulo/server/fs/VolumeManagerImpl.java | 10 ++++++++++
.../apache/accumulo/server/util/ChangeSecret.java | 18 +++++++++++-------
7 files changed, 51 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
index 9dd7b16..08f61d4 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/Volume.java
@@ -45,6 +45,13 @@ public interface Volume {
public Path prefixChild(Path p);
/**
+ * Convert the given child path into a Path that is relative to the base path for this Volume
+ * @param p
+ * @return
+ */
+ public Path prefixChild(String p);
+
+ /**
* Determine if the Path is valid on this Volume (contained by the basePath)
* @param p
* @return True if path is contained within the basePath, false otherwise
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
index 65740ce..0aaf482 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeImpl.java
@@ -48,33 +48,21 @@ public class VolumeImpl implements Volume {
this.basePath = basePath;
}
- /* (non-javadoc)
- * @see org.apache.accumulo.core.volume.Volume#getFileSystem()
- */
@Override
public FileSystem getFileSystem() {
return fs;
}
- /* (non-javadoc)
- * @see org.apache.accumulo.core.volume.Volume#getBasePath()
- */
@Override
public String getBasePath() {
return basePath;
}
- /* (non-javadoc)
- * @see org.apache.accumulo.core.volume.Volume#prefixChild(Path)
- */
@Override
public Path prefixChild(Path p) {
return new Path(basePath, p);
}
- /* (non-javadoc)
- * @see org.apache.accumulo.core.volume.Volume#isValidPath(Path)
- */
@Override
public boolean isValidPath(Path p) {
checkNotNull(p);
@@ -97,4 +85,9 @@ public class VolumeImpl implements Volume {
return getFileSystem() + " " + basePath;
}
+ @Override
+ public Path prefixChild(String p) {
+ return new Path(basePath, p);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 2fa9051..fe1729c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.trace.DistributedTrace;
import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -50,10 +51,11 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
+ Volume defaultVolume = fs.getDefaultVolume();
try {
if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.DATA_VERSION));
- fs.delete(new Path(ServerConstants.getDataVersionLocation() + "/" + ServerConstants.PREV_DATA_VERSION));
+ fs.create(new Path(ServerConstants.getDataVersionLocation(defaultVolume), Integer.toString(ServerConstants.DATA_VERSION)));
+ fs.delete(new Path(ServerConstants.getDataVersionLocation(defaultVolume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
}
} catch (IOException e) {
throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
@@ -76,8 +78,9 @@ public class Accumulo {
}
public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
- Path path = ServerConstants.getDataVersionLocation();
- return getAccumuloPersistentVersion(fs.getFileSystemByPath(path), path);
+ Volume defaultVolume = fs.getDefaultVolume();
+ Path path = ServerConstants.getDataVersionLocation(defaultVolume);
+ return getAccumuloPersistentVersion(defaultVolume.getFileSystem(), path);
}
public static void enableTracing(String address, String application) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 6ea3b91..6c27897 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -27,6 +27,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.conf.ServerConfiguration;
@@ -127,14 +128,14 @@ public class ServerConstants {
return VolumeConfiguration.prefix(getBaseUris(), "walogArchive");
}
- public static Path getInstanceIdLocation() {
+ public static Path getInstanceIdLocation(Volume v) {
// all base dirs should have the same instance id, so can choose any one
- return new Path(getBaseUris()[0], INSTANCE_ID_DIR);
+ return v.prefixChild(INSTANCE_ID_DIR);
}
- public static Path getDataVersionLocation() {
+ public static Path getDataVersionLocation(Volume v) {
// all base dirs should have the same version, so can choose any one
- return new Path(getBaseUris()[0], VERSION_DIR);
+ return v.prefixChild(VERSION_DIR);
}
public static String[] getMetadataTableDirs() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index ce37058..2f7cf42 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.server.fs;
import java.io.IOException;
+import java.util.Collection;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.volume.Volume;
@@ -155,4 +156,9 @@ public interface VolumeManager {
// decide on which of the given locations to create a new file
String choose(String[] options);
+
+ public Volume getDefaultVolume();
+
+ public Collection<Volume> getVolumes();
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index 9b62ce3..e19182d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -546,4 +546,14 @@ public class VolumeManagerImpl implements VolumeManager {
return chooser.choose(options);
}
+ @Override
+ public Volume getDefaultVolume() {
+ return defaultVolume;
+ }
+
+ @Override
+ public Collection<Volume> getVolumes() {
+ return volumesByName.values();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0c841568/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
index ac13034..3f33a0e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ChangeSecret.java
@@ -24,7 +24,7 @@ import java.util.UUID;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
import org.apache.accumulo.fate.zookeeper.ZooReader;
@@ -32,8 +32,9 @@ import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
@@ -57,7 +58,7 @@ public class ChangeSecret {
argsList.add("--new");
argsList.addAll(Arrays.asList(args));
opts.parseArgs(ChangeSecret.class.getName(), argsList.toArray(new String[0]));
- FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+ VolumeManager fs = VolumeManagerImpl.get();
Instance inst = opts.getInstance();
if (!verifyAccumuloIsDown(inst, opts.oldPass))
System.exit(-1);
@@ -142,10 +143,13 @@ public class ChangeSecret {
return newInstanceId;
}
- private static void updateHdfs(FileSystem fs, Instance inst, String newInstanceId) throws IOException {
- fs.delete(ServerConstants.getInstanceIdLocation(), true);
- fs.mkdirs(ServerConstants.getInstanceIdLocation());
- fs.create(new Path(ServerConstants.getInstanceIdLocation(), newInstanceId)).close();
+ private static void updateHdfs(VolumeManager fs, Instance inst, String newInstanceId) throws IOException {
+ // Need to recreate the instanceId on all of them to keep consistency
+ for (Volume v : fs.getVolumes()) {
+ v.getFileSystem().delete(ServerConstants.getInstanceIdLocation(v), true);
+ v.getFileSystem().mkdirs(ServerConstants.getInstanceIdLocation(v));
+ v.getFileSystem().create(new Path(ServerConstants.getInstanceIdLocation(v), newInstanceId)).close();
+ }
}
private static void deleteInstance(Instance origInstance, String oldPass) throws Exception {
[17/23] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --cc proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index abe540a,0000000..efe36b9
mode 100644,000000..100644
--- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@@ -1,1529 -1,0 +1,1567 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.proxy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.DevNull;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.user.SummingCombiner;
+import org.apache.accumulo.core.iterators.user.VersioningIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.examples.simple.constraints.NumericValueConstraint;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
+import org.apache.accumulo.proxy.thrift.AccumuloSecurityException;
+import org.apache.accumulo.proxy.thrift.ActiveCompaction;
+import org.apache.accumulo.proxy.thrift.ActiveScan;
+import org.apache.accumulo.proxy.thrift.BatchScanOptions;
+import org.apache.accumulo.proxy.thrift.Column;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.CompactionReason;
+import org.apache.accumulo.proxy.thrift.CompactionType;
+import org.apache.accumulo.proxy.thrift.Condition;
+import org.apache.accumulo.proxy.thrift.ConditionalStatus;
+import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
+import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
+import org.apache.accumulo.proxy.thrift.DiskUsage;
+import org.apache.accumulo.proxy.thrift.IteratorScope;
+import org.apache.accumulo.proxy.thrift.IteratorSetting;
+import org.apache.accumulo.proxy.thrift.Key;
+import org.apache.accumulo.proxy.thrift.KeyValue;
+import org.apache.accumulo.proxy.thrift.MutationsRejectedException;
+import org.apache.accumulo.proxy.thrift.PartialKey;
+import org.apache.accumulo.proxy.thrift.Range;
+import org.apache.accumulo.proxy.thrift.ScanColumn;
+import org.apache.accumulo.proxy.thrift.ScanOptions;
+import org.apache.accumulo.proxy.thrift.ScanResult;
+import org.apache.accumulo.proxy.thrift.ScanState;
+import org.apache.accumulo.proxy.thrift.ScanType;
+import org.apache.accumulo.proxy.thrift.SystemPermission;
+import org.apache.accumulo.proxy.thrift.TableExistsException;
+import org.apache.accumulo.proxy.thrift.TableNotFoundException;
+import org.apache.accumulo.proxy.thrift.TablePermission;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.accumulo.proxy.thrift.UnknownScanner;
+import org.apache.accumulo.proxy.thrift.UnknownWriter;
+import org.apache.accumulo.proxy.thrift.WriterOptions;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.test.functional.SlowIterator;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.server.TServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+/**
+ * Call every method on the proxy and try to verify that it works.
+ */
+public class SimpleProxyIT {
+
+ public static File macTestFolder = new File(System.getProperty("user.dir") + "/target/" + SimpleProxyIT.class.getName());
+
+ private static MiniAccumuloCluster accumulo;
+ private static String secret = "superSecret";
+ private static Random random = new Random();
+ private static TServer proxyServer;
+ private static Thread thread;
+ private static int proxyPort;
+ private static org.apache.accumulo.proxy.thrift.AccumuloProxy.Client client;
+ private static String principal = "root";
+
+ private static Map<String,String> properties = new TreeMap<String,String>() {
+ private static final long serialVersionUID = 1L;
+
+ {
+ put("password", secret);
+ }
+ };
+ private static ByteBuffer creds = null;
+
+ private static Class<? extends TProtocolFactory> protocolClass;
+
+ static Class<? extends TProtocolFactory> getRandomProtocol() {
+ List<Class<? extends TProtocolFactory>> protocolFactories = new ArrayList<Class<? extends TProtocolFactory>>();
+ protocolFactories.add(org.apache.thrift.protocol.TJSONProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TBinaryProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TTupleProtocol.Factory.class);
+ protocolFactories.add(org.apache.thrift.protocol.TCompactProtocol.Factory.class);
+
+ return protocolFactories.get(random.nextInt(protocolFactories.size()));
+ }
+
+ private static final AtomicInteger tableCounter = new AtomicInteger(0);
+
+ private static String makeTableName() {
+ return "test" + tableCounter.getAndIncrement();
+ }
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder(macTestFolder);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ @BeforeClass
+ public static void setupMiniCluster() throws Exception {
+ FileUtils.deleteQuietly(macTestFolder);
+ macTestFolder.mkdirs();
+ MiniAccumuloConfig config = new MiniAccumuloConfig(macTestFolder, secret).setNumTservers(1);
+ accumulo = new MiniAccumuloCluster(config);
+ accumulo.start();
+
+ Properties props = new Properties();
+ props.put("instance", accumulo.getConfig().getInstanceName());
+ props.put("zookeepers", accumulo.getZooKeepers());
+ props.put("tokenClass", PasswordToken.class.getName());
+
+ protocolClass = getRandomProtocol();
+
+ proxyPort = PortUtils.getRandomFreePort();
+ proxyServer = Proxy.createProxyServer(org.apache.accumulo.proxy.thrift.AccumuloProxy.class, org.apache.accumulo.proxy.ProxyServer.class, proxyPort,
+ protocolClass, props);
+ thread = new Thread() {
+ @Override
+ public void run() {
+ proxyServer.serve();
+ }
+ };
+ thread.start();
+ while (!proxyServer.isServing())
+ UtilWaitThread.sleep(100);
+ client = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ creds = client.login(principal, properties);
+ }
+
+ @Test(timeout = 10000)
+ public void security() throws Exception {
+ client.createLocalUser(creds, "user", s2bb(secret));
+ ByteBuffer badLogin = client.login("user", properties);
+ client.dropLocalUser(creds, "user");
+ final String table = makeTableName();
+ client.createTable(creds, table, false, TimeType.MILLIS);
+
+ final IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+
+ try {
+ client.addConstraint(badLogin, table, NumericValueConstraint.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.addSplits(badLogin, table, Collections.singleton(s2bb("1")));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.clearLocatorCache(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.compactTable(badLogin, table, null, null, null, true, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.cancelCompaction(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createTable(badLogin, table, false, TimeType.MILLIS);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.deleteTable(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.deleteRows(badLogin, table, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.tableExists(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.flushTable(badLogin, table, null, null, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getLocalityGroups(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getMaxRow(badLogin, table, Collections.<ByteBuffer> emptySet(), null, false, null, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getTableProperties(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listSplits(badLogin, table, 10000);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listTables(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.listConstraints(badLogin, table);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.mergeTablets(badLogin, table, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.offlineTable(badLogin, table, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.onlineTable(badLogin, table, false);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeConstraint(badLogin, table, 0);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeTableProperty(badLogin, table, Property.TABLE_FILE_MAX.getKey());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.renameTable(badLogin, table, "someTableName");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ Map<String,Set<String>> groups = new HashMap<String,Set<String>>();
+ groups.put("group1", Collections.singleton("cf1"));
+ groups.put("group2", Collections.singleton("cf2"));
+ client.setLocalityGroups(badLogin, table, groups);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.setTableProperty(badLogin, table, Property.TABLE_FILE_MAX.getKey(), "0");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.tableIdMap(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.getSiteConfiguration(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getSystemConfiguration(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getTabletServers(badLogin);
+ fail("exception not thrown");
+ } catch (TException ex) {}
+ try {
+ client.getActiveScans(badLogin, "fake");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getActiveCompactions(badLogin, "fakse");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeProperty(badLogin, "table.split.threshold");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.setProperty(badLogin, "table.split.threshold", "500M");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.testClassLoad(badLogin, DevNull.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.authenticateUser(badLogin, "root", s2pp(secret));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ HashSet<ByteBuffer> auths = new HashSet<ByteBuffer>(Arrays.asList(s2bb("A"), s2bb("B")));
+ client.changeUserAuthorizations(badLogin, "stooge", auths);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.changeLocalUserPassword(badLogin, "stooge", s2bb(""));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createLocalUser(badLogin, "stooge", s2bb("password"));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.dropLocalUser(badLogin, "stooge");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getUserAuthorizations(badLogin, "stooge");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.grantSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.grantTablePermission(badLogin, "root", table, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.hasSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.hasTablePermission(badLogin, "root", table, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listLocalUsers(badLogin);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.revokeSystemPermission(badLogin, "stooge", SystemPermission.CREATE_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.revokeTablePermission(badLogin, "root", table, TablePermission.ALTER_TABLE);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createScanner(badLogin, table, new ScanOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createBatchScanner(badLogin, table, new BatchScanOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.updateAndFlush(badLogin, table, new HashMap<ByteBuffer,List<ColumnUpdate>>());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createWriter(badLogin, table, new WriterOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.attachIterator(badLogin, "slow", setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.checkIteratorConflicts(badLogin, table, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ final String TABLE_TEST = makeTableName();
+ client.cloneTable(badLogin, table, TABLE_TEST, false, null, null);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.exportTable(badLogin, table, "/tmp");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.importTable(badLogin, "testify", "/tmp");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.getIteratorSetting(badLogin, table, "foo", IteratorScope.SCAN);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.listIterators(badLogin, table);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.removeIterator(badLogin, table, "name", EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.splitRangeByTablets(badLogin, table, client.getRowRange(ByteBuffer.wrap("row".getBytes())), 10);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ File importDir = tempFolder.newFolder("importDir");
+ File failuresDir = tempFolder.newFolder("failuresDir");
+ client.importDirectory(badLogin, table, importDir.getAbsolutePath(), failuresDir.getAbsolutePath(), true);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.pingTabletServer(badLogin, "fake");
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.login("badUser", properties);
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.testTableClassLoad(badLogin, table, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ try {
+ client.createConditionalWriter(badLogin, table, new ConditionalWriterOptions());
+ fail("exception not thrown");
+ } catch (AccumuloSecurityException ex) {}
+ }
+
+ @Test(timeout = 10000)
+ public void tableNotFound() throws Exception {
+ final String doesNotExist = "doesNotExists";
+ try {
+ client.addConstraint(creds, doesNotExist, NumericValueConstraint.class.getName());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.addSplits(creds, doesNotExist, Collections.<ByteBuffer> emptySet());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ final IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
+ try {
+ client.attachIterator(creds, doesNotExist, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.cancelCompaction(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.checkIteratorConflicts(creds, doesNotExist, setting, EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.clearLocatorCache(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ final String TABLE_TEST = makeTableName();
+ client.cloneTable(creds, doesNotExist, TABLE_TEST, false, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.compactTable(creds, doesNotExist, null, null, null, true, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createBatchScanner(creds, doesNotExist, new BatchScanOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createScanner(creds, doesNotExist, new ScanOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createWriter(creds, doesNotExist, new WriterOptions());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.deleteRows(creds, doesNotExist, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.deleteTable(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.exportTable(creds, doesNotExist, "/tmp");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.flushTable(creds, doesNotExist, null, null, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getIteratorSetting(creds, doesNotExist, "foo", IteratorScope.SCAN);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getLocalityGroups(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getMaxRow(creds, doesNotExist, Collections.<ByteBuffer> emptySet(), null, false, null, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getTableProperties(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.grantTablePermission(creds, "root", doesNotExist, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.hasTablePermission(creds, "root", doesNotExist, TablePermission.WRITE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ File importDir = tempFolder.newFolder("importDir");
+ File failuresDir = tempFolder.newFolder("failuresDir");
+ client.importDirectory(creds, doesNotExist, importDir.getAbsolutePath(), failuresDir.getAbsolutePath(), true);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.listConstraints(creds, doesNotExist);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.listSplits(creds, doesNotExist, 10000);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.mergeTablets(creds, doesNotExist, null, null);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.offlineTable(creds, doesNotExist, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.onlineTable(creds, doesNotExist, false);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeConstraint(creds, doesNotExist, 0);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeIterator(creds, doesNotExist, "name", EnumSet.allOf(IteratorScope.class));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.removeTableProperty(creds, doesNotExist, Property.TABLE_FILE_MAX.getKey());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.renameTable(creds, doesNotExist, "someTableName");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.revokeTablePermission(creds, "root", doesNotExist, TablePermission.ALTER_TABLE);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.setTableProperty(creds, doesNotExist, Property.TABLE_FILE_MAX.getKey(), "0");
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.splitRangeByTablets(creds, doesNotExist, client.getRowRange(ByteBuffer.wrap("row".getBytes())), 10);
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.updateAndFlush(creds, doesNotExist, new HashMap<ByteBuffer,List<ColumnUpdate>>());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.getDiskUsage(creds, Collections.singleton(doesNotExist));
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.testTableClassLoad(creds, doesNotExist, VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName());
+ fail("exception not thrown");
+ } catch (TableNotFoundException ex) {}
+ try {
+ client.createConditionalWriter(creds, doesNotExist, new ConditionalWriterOptions());
+ } catch (TableNotFoundException ex) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testExists() throws Exception {
+ client.createTable(creds, "ett1", false, TimeType.MILLIS);
+ client.createTable(creds, "ett2", false, TimeType.MILLIS);
+ try {
+ client.createTable(creds, "ett1", false, TimeType.MILLIS);
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ try {
+ client.renameTable(creds, "ett1", "ett2");
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ try {
+ client.cloneTable(creds, "ett1", "ett2", false, new HashMap<String,String>(), new HashSet<String>());
+ fail("exception not thrown");
+ } catch (TableExistsException tee) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testUnknownScanner() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ String scanner = client.createScanner(creds, TABLE_TEST, null);
+ assertFalse(client.hasNext(scanner));
+ client.closeScanner(scanner);
+
+ try {
+ client.hasNext(scanner);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+
+ try {
+ client.closeScanner(scanner);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+
+ try {
+ client.nextEntry("99999999");
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.nextK("99999999", 6);
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.hasNext("99999999");
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ try {
+ client.hasNext(UUID.randomUUID().toString());
+ fail("exception not thrown");
+ } catch (UnknownScanner us) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testUnknownWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ String writer = client.createWriter(creds, TABLE_TEST, null);
+ client.update(writer, mutation("row0", "cf", "cq", "value"));
+ client.flush(writer);
+ client.update(writer, mutation("row2", "cf", "cq", "value2"));
+ client.closeWriter(writer);
+
+ // this is a oneway call, so it does not throw exceptions
+ client.update(writer, mutation("row2", "cf", "cq", "value2"));
+
+ try {
+ client.flush(writer);
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.flush("99999");
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.flush(UUID.randomUUID().toString());
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ try {
+ client.closeWriter("99999");
+ fail("exception not thrown");
+ } catch (UnknownWriter uw) {}
+ }
+
+ @Test(timeout = 10000)
+ public void testDelete() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row0", "cf", "cq", "value"));
+
+ assertScan(new String[][] {{"row0", "cf", "cq", "value"}}, TABLE_TEST);
+
+ ColumnUpdate upd = new ColumnUpdate(s2bb("cf"), s2bb("cq"));
+ upd.setDeleteCell(true);
+ Map<ByteBuffer,List<ColumnUpdate>> delete = Collections.singletonMap(s2bb("row0"), Collections.singletonList(upd));
+
+ client.updateAndFlush(creds, TABLE_TEST, delete);
+
+ assertScan(new String[][] {}, TABLE_TEST);
+ }
+
- @Test(timeout = 10000)
++ @Test(timeout = 60000)
+ public void testInstanceOperations() throws Exception {
+ int tservers = 0;
+ for (String tserver : client.getTabletServers(creds)) {
+ client.pingTabletServer(creds, tserver);
+ tservers++;
+ }
+ assertTrue(tservers > 0);
+
+ // get something we know is in the site config
+ Map<String,String> cfg = client.getSiteConfiguration(creds);
+ assertTrue(cfg.get("instance.dfs.dir").startsWith(macTestFolder.getPath()));
+
+ // set a property in zookeeper
+ client.setProperty(creds, "table.split.threshold", "500M");
+
+ // check that we can read it
+ for (int i = 0; i < 5; i++) {
+ cfg = client.getSystemConfiguration(creds);
+ if ("500M".equals(cfg.get("table.split.threshold")))
+ break;
+ UtilWaitThread.sleep(200);
+ }
+ assertEquals("500M", cfg.get("table.split.threshold"));
+
+ // unset the setting, check that it's not what it was
+ client.removeProperty(creds, "table.split.threshold");
+ for (int i = 0; i < 5; i++) {
+ cfg = client.getSystemConfiguration(creds);
+ if (!"500M".equals(cfg.get("table.split.threshold")))
+ break;
+ UtilWaitThread.sleep(200);
+ }
+ assertNotEquals("500M", cfg.get("table.split.threshold"));
+
+ // try to load some classes via the proxy
+ assertTrue(client.testClassLoad(creds, DevNull.class.getName(), SortedKeyValueIterator.class.getName()));
+ assertFalse(client.testClassLoad(creds, "foo.bar", SortedKeyValueIterator.class.getName()));
+
+ // create a table that's very slow, so we can look for scans/compactions
+ client.createTable(creds, "slow", true, TimeType.MILLIS);
- IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "200"));
++
++ // Should take 5 seconds to read every record
++ for (int i = 0; i < 20; i++) {
++ client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value"));
++ }
++
++ IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250"));
+ client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class));
- client.updateAndFlush(creds, "slow", mutation("row", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row2", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row3", "cf", "cq", "value"));
- client.updateAndFlush(creds, "slow", mutation("row4", "cf", "cq", "value"));
+
+ // scan
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ String scanner;
+ try {
+ Client client2 = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ scanner = client2.createScanner(creds, "slow", null);
+ client2.nextK(scanner, 10);
+ client2.closeScanner(scanner);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ t.start();
++
+ // look for the scan
- List<ActiveScan> scans = Collections.emptyList();
- loop: for (int i = 0; i < 100; i++) {
++ List<ActiveScan> scans = new ArrayList<ActiveScan>();
++ for (int i = 0; i < 100 && scans.isEmpty(); i++) {
+ for (String tserver : client.getTabletServers(creds)) {
- scans = client.getActiveScans(creds, tserver);
++ List<ActiveScan> scansForServer = client.getActiveScans(creds, tserver);
++ for (ActiveScan scan : scansForServer) {
++ if ("root".equals(scan.getUser())) {
++ scans.add(scan);
++ }
++ }
++
+ if (!scans.isEmpty())
- break loop;
++ break;
+ UtilWaitThread.sleep(10);
+ }
+ }
+ t.join();
++
+ assertFalse(scans.isEmpty());
- ActiveScan scan = scans.get(0);
- assertEquals("root", scan.getUser());
- assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
- assertEquals(ScanType.SINGLE, scan.getType());
- assertEquals("slow", scan.getTable());
- Map<String,String> map = client.tableIdMap(creds);
- assertEquals(map.get("slow"), scan.getExtent().tableId);
- assertTrue(scan.getExtent().endRow == null);
- assertTrue(scan.getExtent().prevEndRow == null);
++ boolean found = false;
++ Map<String,String> map = null;
++ for (int i = 0; i < scans.size() && !found; i++) {
++ ActiveScan scan = scans.get(i);
++ if ("root".equals(scan.getUser())) {
++ assertTrue(ScanState.RUNNING.equals(scan.getState()) || ScanState.QUEUED.equals(scan.getState()));
++ assertEquals(ScanType.SINGLE, scan.getType());
++ assertEquals("slow", scan.getTable());
++
++ map = client.tableIdMap(creds);
++ assertEquals(map.get("slow"), scan.getExtent().tableId);
++ assertTrue(scan.getExtent().endRow == null);
++ assertTrue(scan.getExtent().prevEndRow == null);
++ found = true;
++ }
++ }
++
++ assertTrue("Could not find a scan against the 'slow' table", found);
+
+ // start a compaction
+ t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Client client2 = new TestProxyClient("localhost", proxyPort, protocolClass.newInstance()).proxy();
+ client2.compactTable(creds, "slow", null, null, null, true, true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ t.start();
+
++ final String desiredTableId = map.get("slow");
++
+ // try to catch it in the act
- List<ActiveCompaction> compactions = Collections.emptyList();
- loop2: for (int i = 0; i < 100; i++) {
++ List<ActiveCompaction> compactions = new ArrayList<ActiveCompaction>();
++ for (int i = 0; i < 100 && compactions.isEmpty(); i++) {
++ // Iterate over the tservers
+ for (String tserver : client.getTabletServers(creds)) {
- compactions = client.getActiveCompactions(creds, tserver);
++ // And get the compactions on each
++ List<ActiveCompaction> compactionsOnServer = client.getActiveCompactions(creds, tserver);
++ for (ActiveCompaction compact : compactionsOnServer) {
++ // There might be other compactions occurring (e.g. on METADATA) in which
++ // case we want to prune out those that aren't for our slow table
++ if (desiredTableId.equals(compact.getExtent().tableId)) {
++ compactions.add(compact);
++ }
++ }
++
++ // If we found a compaction for the table we wanted, so we can stop looking
+ if (!compactions.isEmpty())
- break loop2;
++ break;
+ }
+ UtilWaitThread.sleep(10);
+ }
+ t.join();
++
+ // verify the compaction information
+ assertFalse(compactions.isEmpty());
- ActiveCompaction c = compactions.get(0);
- assertEquals(map.get("slow"), c.getExtent().tableId);
- assertTrue(c.inputFiles.isEmpty());
- assertEquals(CompactionType.MINOR, c.getType());
- assertEquals(CompactionReason.USER, c.getReason());
- assertEquals("", c.localityGroup);
- assertTrue(c.outputFile.contains("default_tablet"));
++ for (ActiveCompaction c : compactions) {
++ if (desiredTableId.equals(c.getExtent().tableId)) {
++ assertTrue(c.inputFiles.isEmpty());
++ assertEquals(CompactionType.MINOR, c.getType());
++ assertEquals(CompactionReason.USER, c.getReason());
++ assertEquals("", c.localityGroup);
++ assertTrue(c.outputFile.contains("default_tablet"));
++
++ return;
++ }
++ }
++ fail("Expection to find running compaction for table 'slow' but did not find one");
+ }
+
+ @Test
+ public void testSecurityOperations() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ // check password
+ assertTrue(client.authenticateUser(creds, "root", s2pp(secret)));
+ assertFalse(client.authenticateUser(creds, "root", s2pp("")));
+
+ // create a user
+ client.createLocalUser(creds, "stooge", s2bb("password"));
+ // change auths
+ Set<String> users = client.listLocalUsers(creds);
+ assertEquals(new HashSet<String>(Arrays.asList("root", "stooge")), users);
+ HashSet<ByteBuffer> auths = new HashSet<ByteBuffer>(Arrays.asList(s2bb("A"), s2bb("B")));
+ client.changeUserAuthorizations(creds, "stooge", auths);
+ List<ByteBuffer> update = client.getUserAuthorizations(creds, "stooge");
+ assertEquals(auths, new HashSet<ByteBuffer>(update));
+
+ // change password
+ client.changeLocalUserPassword(creds, "stooge", s2bb(""));
+ assertTrue(client.authenticateUser(creds, "stooge", s2pp("")));
+
+ // check permission failure
+ @SuppressWarnings("serial")
+ ByteBuffer stooge = client.login("stooge", new TreeMap<String,String>() {
+ {
+ put("password", "");
+ }
+ });
+
+ try {
+ client.createTable(stooge, "fail", true, TimeType.MILLIS);
+ fail("should not create the table");
+ } catch (AccumuloSecurityException ex) {
+ assertFalse(client.listTables(creds).contains("fail"));
+ }
+ // grant permissions and test
+ assertFalse(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ client.grantSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE);
+ assertTrue(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ client.createTable(stooge, "success", true, TimeType.MILLIS);
+ client.listTables(creds).contains("succcess");
+
+ // revoke permissions
+ client.revokeSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE);
+ assertFalse(client.hasSystemPermission(creds, "stooge", SystemPermission.CREATE_TABLE));
+ try {
+ client.createTable(stooge, "fail", true, TimeType.MILLIS);
+ fail("should not create the table");
+ } catch (AccumuloSecurityException ex) {
+ assertFalse(client.listTables(creds).contains("fail"));
+ }
+ // create a table to test table permissions
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ // denied!
+ try {
+ String scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 100);
+ fail("stooge should not read table test");
+ } catch (AccumuloSecurityException ex) {}
+ // grant
+ assertFalse(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ client.grantTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ);
+ assertTrue(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ String scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 10);
+ client.closeScanner(scanner);
+ // revoke
+ client.revokeTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ);
+ assertFalse(client.hasTablePermission(creds, "stooge", TABLE_TEST, TablePermission.READ));
+ try {
+ scanner = client.createScanner(stooge, TABLE_TEST, null);
+ client.nextK(scanner, 100);
+ fail("stooge should not read table test");
+ } catch (AccumuloSecurityException ex) {}
+
+ // delete user
+ client.dropLocalUser(creds, "stooge");
+ users = client.listLocalUsers(creds);
+ assertEquals(1, users.size());
+
+ }
+
+ @Test
+ public void testBatchWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+
+ WriterOptions writerOptions = new WriterOptions();
+ writerOptions.setLatencyMs(10000);
+ writerOptions.setMaxMemory(2);
+ writerOptions.setThreads(1);
+ writerOptions.setTimeoutMs(100000);
+
+ String batchWriter = client.createWriter(creds, TABLE_TEST, writerOptions);
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ try {
+ client.flush(batchWriter);
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException ex) {}
+ try {
+ client.closeWriter(batchWriter);
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException e) {}
+
+ client.removeConstraint(creds, TABLE_TEST, 2);
+
+ assertScan(new String[][] {}, TABLE_TEST);
+
+ UtilWaitThread.sleep(2000);
+
+ writerOptions = new WriterOptions();
+ writerOptions.setLatencyMs(10000);
+ writerOptions.setMaxMemory(3000);
+ writerOptions.setThreads(1);
+ writerOptions.setTimeoutMs(100000);
+
+ batchWriter = client.createWriter(creds, TABLE_TEST, writerOptions);
+
+ client.update(batchWriter, mutation("row1", "cf", "cq", "x"));
+ client.flush(batchWriter);
+ client.closeWriter(batchWriter);
+
+ assertScan(new String[][] {{"row1", "cf", "cq", "x"}}, TABLE_TEST);
+
+ client.deleteTable(creds, TABLE_TEST);
+ }
+
+ @Test
+ public void testTableOperations() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ // constraints
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+ assertEquals(2, client.listConstraints(creds, TABLE_TEST).size());
+
+ UtilWaitThread.sleep(2000);
+
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "123"));
+
+ try {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
+ fail("constraint did not fire");
+ } catch (MutationsRejectedException ex) {}
+
+ client.removeConstraint(creds, TABLE_TEST, 2);
+
+ UtilWaitThread.sleep(2000);
+
+ assertEquals(1, client.listConstraints(creds, TABLE_TEST).size());
+
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
+ assertScan(new String[][] {{"row1", "cf", "cq", "x"}}, TABLE_TEST);
+ // splits, merge
+ client.addSplits(creds, TABLE_TEST, new HashSet<ByteBuffer>(Arrays.asList(s2bb("a"), s2bb("m"), s2bb("z"))));
+ List<ByteBuffer> splits = client.listSplits(creds, TABLE_TEST, 1);
+ assertEquals(Arrays.asList(s2bb("m")), splits);
+ client.mergeTablets(creds, TABLE_TEST, null, s2bb("m"));
+ splits = client.listSplits(creds, TABLE_TEST, 10);
+ assertEquals(Arrays.asList(s2bb("m"), s2bb("z")), splits);
+ client.mergeTablets(creds, TABLE_TEST, null, null);
+ splits = client.listSplits(creds, TABLE_TEST, 10);
+ List<ByteBuffer> empty = Collections.emptyList();
+ assertEquals(empty, splits);
+ // iterators
+ client.deleteTable(creds, TABLE_TEST);
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+ HashMap<String,String> options = new HashMap<String,String>();
+ options.put("type", "STRING");
+ options.put("columns", "cf");
+ IteratorSetting setting = new IteratorSetting(10, TABLE_TEST, SummingCombiner.class.getName(), options);
+ client.attachIterator(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
+ for (int i = 0; i < 10; i++) {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "1"));
+ }
+ assertScan(new String[][] {{"row1", "cf", "cq", "10"}}, TABLE_TEST);
+ try {
+ client.checkIteratorConflicts(creds, TABLE_TEST, setting, EnumSet.allOf(IteratorScope.class));
+ fail("checkIteratorConflicts did not throw an exception");
+ } catch (Exception ex) {}
+ client.deleteRows(creds, TABLE_TEST, null, null);
+ client.removeIterator(creds, TABLE_TEST, "test", EnumSet.allOf(IteratorScope.class));
+ String expected[][] = new String[10][];
+ for (int i = 0; i < 10; i++) {
+ client.updateAndFlush(creds, TABLE_TEST, mutation("row" + i, "cf", "cq", "" + i));
+ expected[i] = new String[] {"row" + i, "cf", "cq", "" + i};
+ client.flushTable(creds, TABLE_TEST, null, null, true);
+ }
+ assertScan(expected, TABLE_TEST);
+ // clone
+ final String TABLE_TEST2 = makeTableName();
+ client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
+ assertScan(expected, TABLE_TEST2);
+ client.deleteTable(creds, TABLE_TEST2);
+
+ // don't know how to test this, call it just for fun
+ client.clearLocatorCache(creds, TABLE_TEST);
+
+ // compact
+ client.compactTable(creds, TABLE_TEST, null, null, null, true, true);
+ assertEquals(1, countFiles(TABLE_TEST));
+ assertScan(expected, TABLE_TEST);
+
+ // get disk usage
+ client.cloneTable(creds, TABLE_TEST, TABLE_TEST2, true, null, null);
+ Set<String> tablesToScan = new HashSet<String>();
+ tablesToScan.add(TABLE_TEST);
+ tablesToScan.add(TABLE_TEST2);
+ tablesToScan.add("foo");
+ client.createTable(creds, "foo", true, TimeType.MILLIS);
+ List<DiskUsage> diskUsage = (client.getDiskUsage(creds, tablesToScan));
+ assertEquals(2, diskUsage.size());
+ assertEquals(1, diskUsage.get(0).getTables().size());
+ assertEquals(2, diskUsage.get(1).getTables().size());
+ client.compactTable(creds, TABLE_TEST2, null, null, null, true, true);
+ diskUsage = (client.getDiskUsage(creds, tablesToScan));
+ assertEquals(3, diskUsage.size());
+ assertEquals(1, diskUsage.get(0).getTables().size());
+ assertEquals(1, diskUsage.get(1).getTables().size());
+ assertEquals(1, diskUsage.get(2).getTables().size());
+ client.deleteTable(creds, "foo");
+ client.deleteTable(creds, TABLE_TEST2);
+
+ // export/import
+ File dir = tempFolder.newFolder("test");
+ File destDir = tempFolder.newFolder("test_dest");
+ client.offlineTable(creds, TABLE_TEST, false);
+ client.exportTable(creds, TABLE_TEST, dir.getAbsolutePath());
+ // copy files to a new location
+ FileSystem fs = FileSystem.get(new Configuration());
+ FSDataInputStream is = fs.open(new Path(dir + "/distcp.txt"));
+ BufferedReader r = new BufferedReader(new InputStreamReader(is));
+ while (true) {
+ String line = r.readLine();
+ if (line == null)
+ break;
+ Path srcPath = new Path(line);
+ FileUtils.copyFile(new File(srcPath.toUri().getPath()), new File(destDir, srcPath.getName()));
+ }
+ client.deleteTable(creds, TABLE_TEST);
+ client.importTable(creds, "testify", destDir.getAbsolutePath());
+ assertScan(expected, "testify");
+ client.deleteTable(creds, "testify");
+
+ try {
+ // ACCUMULO-1558 a second import from the same dir should fail, the first import moved the files
+ client.importTable(creds, "testify2", destDir.getAbsolutePath());
+ fail();
+ } catch (Exception e) {}
+
+ assertFalse(client.listTables(creds).contains("testify2"));
+
+ // Locality groups
+ client.createTable(creds, "test", true, TimeType.MILLIS);
+ Map<String,Set<String>> groups = new HashMap<String,Set<String>>();
+ groups.put("group1", Collections.singleton("cf1"));
+ groups.put("group2", Collections.singleton("cf2"));
+ client.setLocalityGroups(creds, "test", groups);
+ assertEquals(groups, client.getLocalityGroups(creds, "test"));
+ // table properties
+ Map<String,String> orig = client.getTableProperties(creds, "test");
+ client.setTableProperty(creds, "test", "table.split.threshold", "500M");
+ Map<String,String> update = client.getTableProperties(creds, "test");
+ assertEquals(update.get("table.split.threshold"), "500M");
+ client.removeTableProperty(creds, "test", "table.split.threshold");
+ update = client.getTableProperties(creds, "test");
+ assertEquals(orig, update);
+ // rename table
+ Map<String,String> tables = client.tableIdMap(creds);
+ client.renameTable(creds, "test", "bar");
+ Map<String,String> tables2 = client.tableIdMap(creds);
+ assertEquals(tables.get("test"), tables2.get("bar"));
+ // table exists
+ assertTrue(client.tableExists(creds, "bar"));
+ assertFalse(client.tableExists(creds, "test"));
+ // bulk import
+ String filename = dir + "/bulk/import/rfile.rf";
+ FileSKVWriter writer = FileOperations.getInstance().openWriter(filename, fs, fs.getConf(), DefaultConfiguration.getInstance());
+ writer.startDefaultLocalityGroup();
+ writer.append(new org.apache.accumulo.core.data.Key(new Text("a"), new Text("b"), new Text("c")), new Value("value".getBytes()));
+ writer.close();
+ fs.mkdirs(new Path(dir + "/bulk/fail"));
+ client.importDirectory(creds, "bar", dir + "/bulk/import", dir + "/bulk/fail", true);
+ String scanner = client.createScanner(creds, "bar", null);
+ ScanResult more = client.nextK(scanner, 100);
+ client.closeScanner(scanner);
+ assertEquals(1, more.results.size());
+ ByteBuffer maxRow = client.getMaxRow(creds, "bar", null, null, false, null, false);
+ assertEquals(s2bb("a"), maxRow);
+
+ assertFalse(client.testTableClassLoad(creds, "bar", "abc123", SortedKeyValueIterator.class.getName()));
+ assertTrue(client.testTableClassLoad(creds, "bar", VersioningIterator.class.getName(), SortedKeyValueIterator.class.getName()));
+ }
+
+ private Condition newCondition(String cf, String cq) {
+ return new Condition(new Column(s2bb(cf), s2bb(cq), s2bb("")));
+ }
+
+ private Condition newCondition(String cf, String cq, String val) {
+ return newCondition(cf, cq).setValue(s2bb(val));
+ }
+
+ private Condition newCondition(String cf, String cq, long ts, String val) {
+ return newCondition(cf, cq).setValue(s2bb(val)).setTimestamp(ts);
+ }
+
+ private ColumnUpdate newColUpdate(String cf, String cq, String val) {
+ return new ColumnUpdate(s2bb(cf), s2bb(cq)).setValue(s2bb(val));
+ }
+
+ private ColumnUpdate newColUpdate(String cf, String cq, long ts, String val) {
+ return new ColumnUpdate(s2bb(cf), s2bb(cq)).setTimestamp(ts).setValue(s2bb(val));
+ }
+
+ private void assertScan(String[][] expected, String table) throws Exception {
+ String scid = client.createScanner(creds, table, new ScanOptions());
+ ScanResult keyValues = client.nextK(scid, expected.length + 1);
+
+ assertEquals(expected.length, keyValues.results.size());
+ assertFalse(keyValues.more);
+
+ for (int i = 0; i < keyValues.results.size(); i++) {
+ checkKey(expected[i][0], expected[i][1], expected[i][2], expected[i][3], keyValues.results.get(i));
+ }
+
+ client.closeScanner(scid);
+ }
+
+ @Test
+ public void testConditionalWriter() throws Exception {
+ final String TABLE_TEST = makeTableName();
+
+ client.createTable(creds, TABLE_TEST, true, TimeType.MILLIS);
+
+ client.addConstraint(creds, TABLE_TEST, NumericValueConstraint.class.getName());
+
+ String cwid = client.createConditionalWriter(creds, TABLE_TEST, new ConditionalWriterOptions());
+
+ Map<ByteBuffer,ConditionalUpdates> updates = new HashMap<ByteBuffer,ConditionalUpdates>();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", 10, "1"),
+ newColUpdate("data", "img", "73435435"))));
+
+ Map<ByteBuffer,ConditionalStatus> results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "73435435"}, {"00345", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test not setting values on conditions
+ updates.clear();
+
+ updates.put(s2bb("00345"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", "2"))));
+ updates.put(s2bb("00346"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq")), Arrays.asList(newColUpdate("meta", "seq", "1"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00345")));
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00346")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "73435435"}, {"00345", "meta", "seq", "1"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting values on conditions
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", "1")), Arrays.asList(newColUpdate("meta", "seq", 20, "2"),
+ newColUpdate("data", "img", "567890"))));
+
+ updates.put(s2bb("00346"), new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", "2")), Arrays.asList(newColUpdate("meta", "seq", "3"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00346")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "567890"}, {"00345", "meta", "seq", "2"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting timestamp on condition to a non-existant version
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", 10, "2")), Arrays.asList(newColUpdate("meta", "seq", 30, "3"),
+ newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "567890"}, {"00345", "meta", "seq", "2"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // test setting timestamp to an existing version
+
+ updates.clear();
+
+ updates.put(
+ s2bb("00345"),
+ new ConditionalUpdates(Arrays.asList(newCondition("meta", "seq", 20, "2")), Arrays.asList(newColUpdate("meta", "seq", 30, "3"),
+ newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00345")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"}}, TABLE_TEST);
+
+ // run test w/ condition that has iterators
+ // following should fail w/o iterator
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+ client.updateAndFlush(creds, TABLE_TEST, Collections.singletonMap(s2bb("00347"), Arrays.asList(newColUpdate("data", "count", "1"))));
+
+ updates.clear();
+ updates.put(s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "count", "3")), Arrays.asList(newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}}, TABLE_TEST);
+
+ // following test w/ iterator setup should succeed
+ Condition iterCond = newCondition("data", "count", "3");
+ Map<String,String> props = new HashMap<String,String>();
+ props.put("type", "STRING");
+ props.put("columns", "data:count");
+ IteratorSetting is = new IteratorSetting(1, "sumc", SummingCombiner.class.getName(), props);
+ iterCond.setIterators(Arrays.asList(is));
+
+ updates.clear();
+ updates.put(s2bb("00347"), new ConditionalUpdates(Arrays.asList(iterCond), Arrays.asList(newColUpdate("data", "img", "1234567890"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // test a mutation that violated a constraint
+ updates.clear();
+ updates.put(s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890")), Arrays.asList(newColUpdate("data", "count", "A"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.VIOLATED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // run test with two conditions
+ // both conditions should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "565"), newCondition("data", "count", "2")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // one condition should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890"), newCondition("data", "count", "2")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // one condition should fail
+ updates.clear();
+ updates.put(
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "565"), newCondition("data", "count", "1")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00347")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "1"}, {"00347", "data", "img", "1234567890"}}, TABLE_TEST);
+
+ // both conditions should succeed
+
+ ConditionalStatus result = client.updateRowConditionally(
+ creds,
+ TABLE_TEST,
+ s2bb("00347"),
+ new ConditionalUpdates(Arrays.asList(newCondition("data", "img", "1234567890"), newCondition("data", "count", "1")), Arrays.asList(
+ newColUpdate("data", "count", "3"), newColUpdate("data", "img", "0987654321"))));
+
+ assertEquals(ConditionalStatus.ACCEPTED, result);
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}}, TABLE_TEST);
+
+ client.closeConditionalWriter(cwid);
+ try {
+ client.updateRowsConditionally(cwid, updates);
+ fail("conditional writer not closed");
+ } catch (UnknownWriter uk) {}
+
+ // run test with colvis
+ client.createLocalUser(creds, "cwuser", s2bb("bestpasswordever"));
+ client.changeUserAuthorizations(creds, "cwuser", Collections.singleton(s2bb("A")));
+ client.grantTablePermission(creds, "cwuser", TABLE_TEST, TablePermission.WRITE);
+ client.grantTablePermission(creds, "cwuser", TABLE_TEST, TablePermission.READ);
+
+ ByteBuffer cwuCreds = client.login("cwuser", Collections.singletonMap("password", "bestpasswordever"));
+
+ cwid = client.createConditionalWriter(cwuCreds, TABLE_TEST, new ConditionalWriterOptions().setAuthorizations(Collections.singleton(s2bb("A"))));
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A")))), Arrays.asList(newColUpdate("data", "seq", "1"),
+ newColUpdate("data", "c", "1").setColVisibility(s2bb("A")))));
+ updates.put(s2bb("00349"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("B")))), Arrays.asList(newColUpdate("data", "seq", "1"))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(2, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00348")));
+ assertEquals(ConditionalStatus.INVISIBLE_VISIBILITY, results.get(s2bb("00349")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "1"}}, TABLE_TEST);
+
+ updates.clear();
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A"))).setValue(s2bb("0"))), Arrays.asList(
+ newColUpdate("data", "seq", "2"), newColUpdate("data", "c", "2").setColVisibility(s2bb("A")))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.REJECTED, results.get(s2bb("00348")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "1"}}, TABLE_TEST);
+
+ updates.clear();
+ updates.put(
+ s2bb("00348"),
+ new ConditionalUpdates(Arrays.asList(new Condition(new Column(s2bb("data"), s2bb("c"), s2bb("A"))).setValue(s2bb("1"))), Arrays.asList(
+ newColUpdate("data", "seq", "2"), newColUpdate("data", "c", "2").setColVisibility(s2bb("A")))));
+
+ results = client.updateRowsConditionally(cwid, updates);
+
+ assertEquals(1, results.size());
+ assertEquals(ConditionalStatus.ACCEPTED, results.get(s2bb("00348")));
+
+ assertScan(new String[][] { {"00345", "data", "img", "1234567890"}, {"00345", "meta", "seq", "3"}, {"00346", "meta", "seq", "1"},
+ {"00347", "data", "count", "3"}, {"00347", "data", "img", "0987654321"}, {"00348", "data", "seq", "2"}}, TABLE_TEST);
+
+ client.closeConditionalWriter(cwid);
+ try {
+ client.updateRowsConditionally(cwid, updates);
+ fail("conditional writer not closed");
+ } catch (UnknownWriter uk) {}
+
+ client.dropLocalUser(creds, "cwuser");
+
+ }
+
+ private void checkKey(String row, String cf, String cq, String val, KeyValue keyValue) {
+ assertEquals(row, ByteBufferUtil.toString(keyValue.key.row));
+ assertEquals(cf, ByteBufferUtil.toString(keyValue.key.colFamily));
+ assertEquals(cq, ByteBufferUtil.toString(keyValue.key.colQualifier));
+ assertEquals("", ByteBufferUtil.toString(keyValue.key.colVisibility));
+ assertEquals(val, ByteBufferUtil.toString(keyValue.value));
+ }
+
+ // scan metadata for file entries for the given table
+ private int countFiles(String table) throws Exception {
+ Map<String,String> tableIdMap = client.tableIdMap(creds);
+ String tableId = tableIdMap.get(table);
+ Key start = new Key();
+ start.row = s2bb(tableId + ";");
+ Key end = new Key();
+ end.row = s2bb(tableId + "<");
+ end = client.getFollowing(end, PartialKey.ROW);
+ ScanOptions opt = new ScanOptions();
+ opt.range = new Range(start, true, end, false);
+ opt.columns = Collections.singletonList(new ScanColumn(s2bb("file")));
+ String scanner = client.createScanner(creds, MetadataTable.NAME, opt);
+ int result = 0;
+ while (true) {
+ ScanResult more = client.nextK(scanner, 100);
+ result += more.getResults().size();
+ if (!more.more)
+ break;
+ }
+ return result;
+ }
+
+ private Map<ByteBuffer,List<ColumnUpdate>> mutation(String row, String cf, String cq, String value) {
+ ColumnUpdate upd = new ColumnUpdate(s2bb(cf), s2bb(cq));
+ upd.setValue(value.getBytes());
+ return Collections.singletonMap(s2bb(row), Collections.singletonList(upd));
+ }
+
+ private ByteBuffer s2bb(String cf) {
+ return ByteBuffer.wrap(cf.getBytes());
+ }
+
+ private Map<String,String> s2pp(String cf) {
+ Map<String,String> toRet = new TreeMap<String,String>();
+ toRet.put("password", cf);
+ return toRet;
+ }
+
+ static private ByteBuffer t2bb(Text t) {
+ return ByteBuffer.wrap(t.getBytes());
+ }
+
+ @Test
+ public void testGetRowRange() throws Exception {
+ Range range = client.getRowRange(s2bb("xyzzy"));
+ org.apache.accumulo.core.data.Range range2 = new org.apache.accumulo.core.data.Range(new Text("xyzzy"));
+ assertEquals(0, range.start.row.compareTo(t2bb(range2.getStartKey().getRow())));
+ assertEquals(0, range.stop.row.compareTo(t2bb(range2.getEndKey().getRow())));
+ assertEquals(range.startInclusive, range2.isStartKeyInclusive());
+ assertEquals(range.stopInclusive, range2.isEndKeyInclusive());
+ assertEquals(0, range.start.colFamily.compareTo(t2bb(range2.getStartKey().getColumnFamily())));
+ assertEquals(0, range.start.colQualifier.compareTo(t2bb(range2.getStartKey().getColumnQualifier())));
+ assertEquals(0, range.stop.colFamily.compareTo(t2bb(range2.getEndKey().getColumnFamily())));
+ assertEquals(0, range.stop.colQualifier.compareTo(t2bb(range2.getEndKey().getColumnQualifier())));
+ assertEquals(range.start.timestamp, range.start.timestamp);
+ assertEquals(range.stop.timestamp, range.stop.timestamp);
+ }
+
+ @AfterClass
+ public static void tearDownMiniCluster() throws Exception {
+ accumulo.stop();
+ }
+}
[23/23] git commit: ACCUMULO-2061 Update FileTypeTest to assert that
volumes don't need to have an 'accumulo' directory
Posted by el...@apache.org.
ACCUMULO-2061 Update FileTypeTest to assert that volumes don't need to have an 'accumulo' directory
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2da21b6a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2da21b6a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2da21b6a
Branch: refs/heads/ACCUMULO-2061
Commit: 2da21b6a450c474115ad8d9c94747116dccf4017
Parents: 9060185
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 14:09:28 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 14:09:28 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/fs/FileTypeTest.java | 48 ++++++++++++++------
1 file changed, 35 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2da21b6a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
index 205a793..ad29c19 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java
@@ -27,28 +27,50 @@ import org.junit.Test;
public class FileTypeTest {
@Test
public void testVolumeExtraction() {
- Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/accumulo/wal/1.2.3.4/aaa-bbb-ccc-ddd")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf")));
+
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/wal/1.2.3.4/aaa-bbb-ccc-ddd")));
Assert.assertNull(FileType.WAL.getVolume(new Path("1.2.3.4/aaa-bbb-ccc-ddd")));
Assert.assertNull(FileType.TABLE.getVolume(new Path("../2b/t-001/C00.rf")));
Assert.assertNull(FileType.TABLE.getVolume(new Path("/t-001/C00.rf")));
- Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("hdfs://nn1/a/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/a/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("hdfs://nn1/a"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf")));
+
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf")));
- Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf")));
+ // Having an 'accumulo' directory is not a requirement
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf")));
+ Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf")));
}
}
[11/23] git commit: ACCUMULO-2005: Add a 3rd argument to functional
test RunTests.java to allow a timeout factor for mapred.task.timeout and
run.py -f flag
Posted by el...@apache.org.
ACCUMULO-2005: Add a 3rd argument to functional test RunTests.java to allow a timeout factor for mapred.task.timeout and run.py -f flag
(reimplemented for 1.5.x branch)
Signed-off-by: Sean Busbey <bu...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bc9cee0c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bc9cee0c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bc9cee0c
Branch: refs/heads/ACCUMULO-2061
Commit: bc9cee0c02952a0ba6d4c96f71204e84a5acb701
Parents: 32dc104
Author: Hung Pham <hp...@cloudera.com>
Authored: Sun Feb 9 18:07:30 2014 -0500
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Mon Mar 10 09:23:32 2014 -0500
----------------------------------------------------------------------
.../accumulo/test/functional/RunTests.java | 21 +++++++++++++++++---
test/system/auto/README | 10 ++++++++++
2 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc9cee0c/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
index 0f4dd8e..b70afd1 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
@@ -76,19 +76,26 @@ public class RunTests extends Configured implements Tool {
private static final Logger log = Logger.getLogger(RunTests.class);
private Job job = null;
-
+
+ private static final int DEFAULT_TIMEOUT_FACTOR = 1;
+
static class Opts extends Help {
@Parameter(names="--tests", description="newline separated list of tests to run", required=true)
String testFile;
@Parameter(names="--output", description="destination for the results of tests in HDFS", required=true)
String outputPath;
+ @Parameter(names="--timeoutFactor", description="Optional scaling factor for timeout for both mapred.task.timeout and -f flag on run.py", required=false)
+ Integer intTimeoutFactor = DEFAULT_TIMEOUT_FACTOR;
}
+ static final String TIMEOUT_FACTOR = RunTests.class.getName() + ".timeoutFactor";
+
static public class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
private static final String REDUCER_RESULT_START = "::::: ";
private static final int RRS_LEN = REDUCER_RESULT_START.length();
private Text result = new Text();
+ String mapperTimeoutFactor = null;
private static enum Outcome {
SUCCESS, FAILURE, ERROR, UNEXPECTED_SUCCESS, EXPECTED_FAILURE
@@ -105,7 +112,7 @@ public class RunTests extends Configured implements Tool {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
- List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-t", value.toString());
+ List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-f", mapperTimeoutFactor, "-t", value.toString());
log.info("Running test " + cmd);
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.directory(new File(context.getConfiguration().get("accumulo.home")));
@@ -140,6 +147,10 @@ public class RunTests extends Configured implements Tool {
p.waitFor();
}
+ @Override
+ protected void setup(Mapper.Context context) throws IOException, InterruptedException {
+ mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, DEFAULT_TIMEOUT_FACTOR));
+ }
}
@Override
@@ -153,7 +164,11 @@ public class RunTests extends Configured implements Tool {
Configuration conf = job.getConfiguration();
conf.setInt("mapred.max.split.size", 40);
conf.set("accumulo.home", System.getenv("ACCUMULO_HOME"));
- conf.setInt("mapred.task.timeout", 8 * 60 * 1000);
+
+ // Taking third argument as scaling factor to setting mapred.task.timeout
+ // and TIMEOUT_FACTOR
+ conf.setInt("mapred.task.timeout", opts.intTimeoutFactor * 8 * 60 * 1000);
+ conf.setInt(TIMEOUT_FACTOR, opts.intTimeoutFactor);
conf.setBoolean("mapred.map.tasks.speculative.execution", false);
// set input
http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc9cee0c/test/system/auto/README
----------------------------------------------------------------------
diff --git a/test/system/auto/README b/test/system/auto/README
index 41f6159..c12ca68 100644
--- a/test/system/auto/README
+++ b/test/system/auto/README
@@ -69,6 +69,16 @@ cluster at your disposal, you can run the tests as a MapReduce job:
The example above runs every test. You can trim the tests file to include
only the tests you wish to run.
+You may specify a 'timeout factor' via an optional integer argument:
+
+ $ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \
+ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor <integer>
+
+Where 'timeout_factor' indicates how much we should scale up timeouts. It will
+be used to set both mapred.task.timeout and the "-f" flag used by run.py. If
+not given, 'timeout_factor' defaults to 1, which corresponds to a
+mapred.task.timeout of 480 seconds.
+
In some clusters, the user under which MR jobs run is different from the user
under which Accumulo is installed, and this can cause failures running the
tests. Various configuration and permission changes can be made to help the
[22/23] git commit: ACCUMULO-2061 Don't use the default volume,
use one of the "real" volumes.
Posted by el...@apache.org.
ACCUMULO-2061 Don't use the default volume, use one of the "real" volumes.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9060185c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9060185c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9060185c
Branch: refs/heads/ACCUMULO-2061
Commit: 9060185c500b2ab17b99b09c1e1c3cd1791b1c6e
Parents: 039989a
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 14:00:06 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 14:00:06 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/Accumulo.java | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9060185c/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index fe1729c..48534f0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -51,11 +51,12 @@ public class Accumulo {
private static final Logger log = Logger.getLogger(Accumulo.class);
public static synchronized void updateAccumuloVersion(VolumeManager fs) {
- Volume defaultVolume = fs.getDefaultVolume();
+ // TODO ACCUMULO-2451 Should update all volumes, not one
+ Volume volume = fs.getVolumes().iterator().next();
try {
if (getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) {
- fs.create(new Path(ServerConstants.getDataVersionLocation(defaultVolume), Integer.toString(ServerConstants.DATA_VERSION)));
- fs.delete(new Path(ServerConstants.getDataVersionLocation(defaultVolume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
+ fs.create(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.DATA_VERSION)));
+ fs.delete(new Path(ServerConstants.getDataVersionLocation(volume), Integer.toString(ServerConstants.PREV_DATA_VERSION)));
}
} catch (IOException e) {
throw new RuntimeException("Unable to set accumulo version: an error occurred.", e);
@@ -78,9 +79,10 @@ public class Accumulo {
}
public static synchronized int getAccumuloPersistentVersion(VolumeManager fs) {
- Volume defaultVolume = fs.getDefaultVolume();
- Path path = ServerConstants.getDataVersionLocation(defaultVolume);
- return getAccumuloPersistentVersion(defaultVolume.getFileSystem(), path);
+ // It doesn't matter which Volume is used as they should all have the data version stored
+ Volume v = fs.getVolumes().iterator().next();
+ Path path = ServerConstants.getDataVersionLocation(v);
+ return getAccumuloPersistentVersion(v.getFileSystem(), path);
}
public static void enableTracing(String address, String application) {
[21/23] git commit: ACCUMULO-2061 Remove implicit assumptions about
'/accumulo' directory that all volumes would have.
Posted by el...@apache.org.
ACCUMULO-2061 Remove implicit assumptions about '/accumulo' directory that all volumes would have.
Added some extra logging at trace to help if this needs to be revisited.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/039989ad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/039989ad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/039989ad
Branch: refs/heads/ACCUMULO-2061
Commit: 039989ad44bb9b46d620a3e302e15c32b3b3840a
Parents: c3214e0
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 13:57:48 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 13:57:48 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/ServerConstants.java | 4 +-
.../accumulo/server/fs/VolumeManager.java | 3 +-
.../apache/accumulo/server/fs/VolumeUtil.java | 20 ++++-
.../accumulo/server/fs/VolumeUtilTest.java | 89 ++++++++++++++++++--
4 files changed, 103 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
index 6c27897..7dd0a08 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
@@ -187,8 +187,8 @@ public class ServerConstants {
HashSet<Path> baseDirs = new HashSet<Path>();
for (String baseDir : getBaseUris()) {
- // normalize using path and remove accumulo dir
- baseDirs.add(new Path(baseDir).getParent());
+ // normalize using path
+ baseDirs.add(new Path(baseDir));
}
for (Pair<Path,Path> pair : ret)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index c345ede..9b8fb98 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -48,9 +48,10 @@ public interface VolumeManager {
}
private static int endOfVolumeIndex(String path, String dir) {
+ // Strip off the suffix that starts with the FileType (e.g. tables, wal, etc)
int dirIndex = path.indexOf('/' + dir);
if (dirIndex != -1) {
- return path.lastIndexOf('/', dirIndex - 1);
+ return dirIndex;
}
if (path.contains(":"))
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 110f1e4..bcfb008 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -81,8 +81,10 @@ public class VolumeUtil {
}
public static String switchVolume(String path, FileType ft, List<Pair<Path,Path>> replacements) {
- if (replacements.size() == 0)
+ if (replacements.size() == 0) {
+ log.trace("Not switching volume because there are no replacements");
return null;
+ }
Path p = new Path(path);
@@ -92,10 +94,15 @@ public class VolumeUtil {
for (Pair<Path,Path> pair : replacements) {
Path key = removeTrailingSlash(pair.getFirst());
- if (key.equals(volume))
- return new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ if (key.equals(volume)) {
+ String replacement = new Path(pair.getSecond(), ft.removeVolume(p)).toString();
+ log.trace("Replacing " + path + " with " + replacement);
+ return replacement;
+ }
}
+ log.trace("Could not find replacement for " + ft + " at " + path);
+
return null;
}
@@ -119,13 +126,17 @@ public class VolumeUtil {
}
- if (numSwitched == 0)
+ if (numSwitched == 0) {
+ log.trace("Did not switch " + le);
return null;
+ }
LogEntry newLogEntry = new LogEntry(le);
newLogEntry.filename = switchedPath;
newLogEntry.logSet = switchedLogs;
+ log.trace("Switched " + le + " to " + newLogEntry);
+
return newLogEntry;
}
@@ -165,6 +176,7 @@ public class VolumeUtil {
*/
public static TabletFiles updateTabletVolumes(ZooLock zooLock, VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles) throws IOException {
List<Pair<Path,Path>> replacements = ServerConstants.getVolumeReplacements();
+ log.trace("Using volume replacements: " + replacements);
List<LogEntry> logsToRemove = new ArrayList<LogEntry>();
List<LogEntry> logsToAdd = new ArrayList<LogEntry>();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
index c85be45..3b905c9 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java
@@ -43,9 +43,9 @@ public class VolumeUtilTest {
@Test
public void testSwitchVolume() {
List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -57,9 +57,9 @@ public class VolumeUtilTest {
Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
replacements.clear();
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a")));
- replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a/accumulo")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo")));
Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf",
VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
@@ -73,6 +73,70 @@ public class VolumeUtilTest {
}
@Test
+ public void testSwitchVolumesDifferentSourceDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b")));
+
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
+ public void testSwitchVolumesDifferentTargetDepths() {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+
+ replacements.clear();
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/path1/path2")));
+ replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3")));
+
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf",
+ VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements));
+ }
+
+ @Test
public void testSame() throws Exception {
FileSystem fs = FileSystem.getLocal(new Configuration());
@@ -133,6 +197,19 @@ public class VolumeUtilTest {
}
+ @Test
+ public void testRootTableReplacement() throws IOException {
+ List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>();
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8")));
+ replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9")));
+
+ FileType ft = FileType.TABLE;
+
+ Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet",
+ VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet",
+ ft, replacements));
+ }
+
private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException {
FSDataOutputStream out = fs.create(new Path(dir, filename));
try {
[07/23] git commit: ACCUMULO-2438 Backport the changes to ensure that
we watch separate sets of tables for merges
Posted by el...@apache.org.
ACCUMULO-2438 Backport the changes to ensure that we watch separate sets of tables for merges
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/12b53b04
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/12b53b04
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/12b53b04
Branch: refs/heads/ACCUMULO-2061
Commit: 12b53b043b60368fb647db9c0a15916171d4504b
Parents: 18006d2
Author: Josh Elser <el...@apache.org>
Authored: Sun Mar 9 23:34:18 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sun Mar 9 23:48:44 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/master/Master.java | 9 ++-
.../apache/accumulo/test/DeleteRowsTest.java | 78 ++++++++++++++++++++
2 files changed, 85 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/12b53b04/server/src/main/java/org/apache/accumulo/server/master/Master.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/master/Master.java b/server/src/main/java/org/apache/accumulo/server/master/Master.java
index e72fe3f..8c4c864 100644
--- a/server/src/main/java/org/apache/accumulo/server/master/Master.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/Master.java
@@ -1288,9 +1288,10 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
int unloaded = 0;
try {
Map<Text,MergeStats> mergeStatsCache = new HashMap<Text,MergeStats>();
+ Map<Text,MergeStats> currentMerges = new HashMap<Text,MergeStats>();
for (MergeInfo merge : merges()) {
if (merge.getRange() != null) {
- mergeStatsCache.put(merge.getRange().getTableId(), new MergeStats(merge));
+ currentMerges.put(merge.getRange().getTableId(), new MergeStats(merge));
}
}
@@ -1339,7 +1340,11 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
Text tableId = tls.extent.getTableId();
MergeStats mergeStats = mergeStatsCache.get(tableId);
if (mergeStats == null) {
- mergeStatsCache.put(tableId, mergeStats = new MergeStats(new MergeInfo()));
+ mergeStats = currentMerges.get(tableId);
+ if (mergeStats == null) {
+ mergeStats = new MergeStats(new MergeInfo());
+ }
+ mergeStatsCache.put(tableId, mergeStats);
}
TabletGoalState goal = getGoalState(tls, mergeStats.getMergeInfo());
TServerInstance server = tls.getServer();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/12b53b04/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java b/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
new file mode 100644
index 0000000..a627dc1
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/DeleteRowsTest.java
@@ -0,0 +1,78 @@
+/*
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.MiniAccumuloCluster;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.Iterables;
+
+public class DeleteRowsTest {
+ private static final Logger log = Logger.getLogger(DeleteRowsTest.class);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private static String secret = "superSecret";
+ public static TemporaryFolder folder = new TemporaryFolder();
+ public static MiniAccumuloCluster cluster;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ folder.create();
+ MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret);
+ cfg.setNumTservers(1);
+ cluster = new MiniAccumuloCluster(cfg);
+ cluster.start();
+
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ cluster.stop();
+ folder.delete();
+ }
+
+ @Test(timeout = 120 * 1000)
+ public void test() throws Exception {
+ ZooKeeperInstance zk = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers());
+ Connector c = zk.getConnector("root", new PasswordToken(secret));
+ for (int i = 0; i < 20; i++) {
+ final String tableName = testName.getMethodName() + i;
+ log.debug("Creating " + tableName);
+ c.tableOperations().create(tableName);
+ log.debug("Deleting rows from " + tableName);
+ c.tableOperations().deleteRows(tableName, null, null);
+ log.debug("Verifying no rows were found");
+ Scanner scanner = c.createScanner(tableName, new Authorizations());
+ assertEquals(0, Iterables.size(scanner));
+ }
+ }
+}
[05/23] git commit: ACCUMULO-2441 outline the file prefix conventions
Posted by el...@apache.org.
ACCUMULO-2441 outline the file prefix conventions
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0297276e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0297276e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0297276e
Branch: refs/heads/ACCUMULO-2061
Commit: 0297276e692d117cd515ec31d1ca1412570e4785
Parents: 4fabfba
Author: Eric Newton <er...@gmail.com>
Authored: Fri Mar 7 19:05:56 2014 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Fri Mar 7 19:05:56 2014 -0500
----------------------------------------------------------------------
.../chapters/troubleshooting.tex | 29 ++++++++++++++++++++
1 file changed, 29 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0297276e/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex b/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
index 8ba7176..18d472f 100644
--- a/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
+++ b/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
@@ -599,3 +599,32 @@ but the basic approach is:
\item Recreate tables, users and permissions
\item Import the directories under \texttt{/corrupt/tables/<id>} into the new instance
\end{itemize}
+
+\section{File Naming Conventions}
+
+Q. Why are files named like they are? Why do some start with ``C'' and others with ``F''?
+
+A. The file names give you a basic idea for the source of the file.
+
+The base of the filename is a base-36 unique number. All filenames in accumulo are coordinated
+with a counter in zookeeper, so they are always unique, which is useful for debugging.
+
+The leading letter gives you an idea of how the file was created:
+
+\begin{itemize}
+ \item F - Flush: entries in memory were written to a file (Minor Compaction)
+ \item M - Merging compaction: entries in memory were combined with the smallest file to create one new file
+ \item C - Several files, but not all files, were combined to produce this file (Major Compaction)
+ \item A - All files were compacted, delete entries were dropped
+ \item I - Bulk import, complete, sorted index files. Always in a directory starting with "b-"
+\end{itemize}
+
+This simple file naming convention allows you to see the basic structure of the files from just
+their filenames, and reason about what should be happening to them next, just
+by scanning their entries in the metadata tables.
+
+For example, if you see multiple files with ``M'' prefixes, the tablet is, or was, up against it's
+maximum file limit, so it began merging memory updates with files to keep the file count reasonable. This
+slows down ingest performance, so knowing there are many files like this tells you that the system
+is struggling to keep up with ingest vs the compaction strategy which reduces the number of files.
+
[18/23] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/da7f937b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/da7f937b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/da7f937b
Branch: refs/heads/ACCUMULO-2061
Commit: da7f937b1d7d88ba7508b22a92ff7d0d110954ae
Parents: 4223d66 4cc25ca
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 18:15:28 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 18:15:28 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +--
.../core/util/shell/ShellConfigTest.java | 5 +-
.../core/util/shell/ShellSetInstanceTest.java | 3 +-
.../accumulo/core/util/shell/ShellTest.java | 5 +-
.../apache/accumulo/proxy/SimpleProxyIT.java | 98 +++++++++----
.../org/apache/accumulo/test/ShellServerIT.java | 145 ++++++++++++++-----
6 files changed, 196 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
index 420d465,6943b37..0c2cf3c
--- a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
@@@ -219,111 -200,114 +219,109 @@@ public class Shell extends ShellOption
private boolean logErrorsToConsole = false;
private PrintWriter writer = null;
private boolean masking = false;
-
+
public Shell() throws IOException {
- this(new ConsoleReader());
+ this(new ConsoleReader(), new PrintWriter(
+ new OutputStreamWriter(System.out,
+ System.getProperty("jline.WindowsTerminal.output.encoding", System.getProperty("file.encoding")))));
}
-
+
- public Shell(ConsoleReader reader) {
+ public Shell(ConsoleReader reader, PrintWriter writer) {
super();
this.reader = reader;
- }
-
- public Shell(ConsoleReader reader, PrintWriter writer) {
- this(reader);
this.writer = writer;
}
-
+
// Not for client use
public boolean config(String... args) {
-
- CommandLine cl;
+ ShellOptionsJC options = new ShellOptionsJC();
+ JCommander jc = new JCommander();
+
+ jc.setProgramName("accumulo shell");
+ jc.addObject(options);
try {
- cl = new BasicParser().parse(opts, args);
- if (cl.getArgs().length > 0)
- throw new ParseException("Unrecognized arguments: " + cl.getArgList());
-
- if (cl.hasOption(helpOpt.getOpt())) {
- configError = true;
- printHelp("shell", SHELL_DESCRIPTION, opts);
- return true;
- }
-
- setDebugging(cl.hasOption(debugOption.getLongOpt()));
- authTimeout = Integer.parseInt(cl.getOptionValue(authTimeoutOpt.getLongOpt(), DEFAULT_AUTH_TIMEOUT)) * 60 * 1000;
- disableAuthTimeout = cl.hasOption(disableAuthTimeoutOpt.getLongOpt());
-
- if (cl.hasOption(zooKeeperInstance.getOpt()) && cl.getOptionValues(zooKeeperInstance.getOpt()).length != 2)
- throw new MissingArgumentException(zooKeeperInstance);
-
- } catch (Exception e) {
+ jc.parse(args);
+ } catch (ParameterException e) {
configError = true;
- printException(e);
- printHelp("shell", SHELL_DESCRIPTION, opts);
+ }
+
+ if (options.isHelpEnabled()) {
+ configError = true;
+ }
+
+ if (!configError && options.getUnrecognizedOptions() != null) {
+ configError = true;
+ logError("Unrecognized Options: " + options.getUnrecognizedOptions().toString());
+ }
+
+ if (configError) {
+ jc.usage();
return true;
}
-
+
+ setDebugging(options.isDebugEnabled());
+ authTimeout = options.getAuthTimeout() * 60 * 1000; // convert minutes to milliseconds
+ disableAuthTimeout = options.isAuthTimeoutDisabled();
+
// get the options that were parsed
- String sysUser = System.getProperty("user.name");
- if (sysUser == null)
- sysUser = "root";
- String user = cl.getOptionValue(usernameOption.getOpt(), sysUser);
-
- String passw = cl.getOptionValue(passwOption.getOpt(), null);
- tabCompletion = !cl.hasOption(tabCompleteOption.getLongOpt());
- String[] loginOptions = cl.getOptionValues(loginOption.getOpt());
-
+ String user = options.getUsername();
+ String password = options.getPassword();
+
+ tabCompletion = !options.isTabCompletionDisabled();
+
// Use a fake (Mock), ZK, or HdfsZK Accumulo instance
- setInstance(cl);
-
+ setInstance(options);
+
+ // AuthenticationToken options
+ token = options.getAuthenticationToken();
+ Map<String,String> loginOptions = options.getTokenProperties();
+
// process default parameters if unspecified
try {
- if (loginOptions != null && !cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Must supply '-" + tokenOption.getOpt() + "' option with '-" + loginOption.getOpt() + "' option");
-
- if (loginOptions == null && cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Must supply '-" + loginOption.getOpt() + "' option with '-" + tokenOption.getOpt() + "' option");
-
- if (passw != null && cl.hasOption(tokenOption.getOpt()))
- throw new IllegalArgumentException("Can not supply '-" + passwOption.getOpt() + "' option with '-" + tokenOption.getOpt() + "' option");
-
- if (user == null)
- throw new MissingArgumentException(usernameOption);
-
- if (loginOptions != null && cl.hasOption(tokenOption.getOpt())) {
- Properties props = new Properties();
- for (String loginOption : loginOptions)
- for (String lo : loginOption.split(",")) {
- String[] split = lo.split("=");
- props.put(split[0], split[1]);
- }
-
- this.token = Class.forName(cl.getOptionValue(tokenOption.getOpt())).asSubclass(AuthenticationToken.class).newInstance();
- this.token.init(props);
- }
-
- if (!cl.hasOption(fakeOption.getLongOpt())) {
- DistributedTrace.enable(instance, new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()), "shell", InetAddress.getLocalHost()
- .getHostName());
+ boolean hasToken = (token != null);
+ boolean hasTokenOptions = !loginOptions.isEmpty();
+
+ if (hasToken && password != null) {
+ throw new ParameterException("Can not supply '--pass' option with '--tokenClass' option");
}
-
+
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
- public void start() {
- reader.getTerminal().enableEcho();
+ public void run() {
+ reader.getTerminal().setEchoEnabled(true);
}
});
-
- if (passw != null) {
- this.token = new PasswordToken(passw);
+
+ // Need either both a token and options, or neither, but not just one.
+ if (hasToken != hasTokenOptions) {
+ throw new ParameterException("Must supply either both or neither of '--tokenClass' and '--tokenProperty'");
+ } else if (hasToken) { // implied hasTokenOptions
+ // Fully qualified name so we don't shadow java.util.Properties
+ org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties props;
+ // and line wrap it because the package name is so long
+ props = new org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties();
+
+ props.putAllStrings(loginOptions);
+ token.init(props);
+ } else {
+ // Read password if the user explicitly asked for it, or didn't specify anything at all
+ if ("stdin".equals(password) || password == null) {
+ password = reader.readLine("Password: ", '*');
+ }
+
+ if (password == null) {
+ // User cancel, e.g. Ctrl-D pressed
+ throw new ParameterException("No password or token option supplied");
+ } else {
+ this.token = new PasswordToken(password);
+ }
}
-
- if (this.token == null) {
- passw = readMaskedLine("Password: ", '*');
- if (passw != null)
- this.token = new PasswordToken(passw);
+
+ if (!options.isFake()) {
+ ZooReader zr = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ DistributedTrace.enable(instance, zr, "shell", InetAddress.getLocalHost().getHostName());
}
-
- if (this.token == null) {
- reader.printNewline();
- throw new MissingArgumentException("No password or token option supplied");
- } // user canceled
-
+
this.setTableName("");
this.principal = user;
connector = instance.getConnector(this.principal, token);
@@@ -1053,17 -939,13 +1051,13 @@@
private final void printHelp(String usage, String description, Options opts) {
printHelp(usage, description, opts, Integer.MAX_VALUE);
}
-
+
private final void printHelp(String usage, String description, Options opts, int width) {
- PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, Constants.UTF8));
- new HelpFormatter().printHelp(pw, width, usage, description, opts, 2, 5, null, true);
- pw.flush();
- if (logErrorsToConsole && writer != null) {
- new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
- writer.flush();
- }
+ // TODO Use the OutputStream from the JLine ConsoleReader if we can ever get access to it
+ new HelpFormatter().printHelp(writer, width, usage, description, opts, 2, 5, null, true);
+ writer.flush();
}
-
+
public int getExitCode() {
return exitCode;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
index c9914da,0000000..43d2e12
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellConfigTest.java
@@@ -1,89 -1,0 +1,90 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.PrintStream;
++import java.io.PrintWriter;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.util.shell.ShellTest.TestOutputStream;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.beust.jcommander.ParameterException;
+
+public class ShellConfigTest {
+ TestOutputStream output;
+ Shell shell;
+ PrintStream out;
+
+ @Before
+ public void setUp() throws Exception {
+ Shell.log.setLevel(Level.ERROR);
+
+ out = System.out;
+ output = new TestOutputStream();
+ System.setOut(new PrintStream(output));
-
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output));
++
++ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output), new PrintWriter(output));
+ shell.setLogErrorsToConsole();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ shell.shutdown();
+ output.clear();
+ System.setOut(out);
+ }
+
+ @Test
+ public void testHelp() {
+ assertTrue(shell.config("--help"));
+ assertTrue("Did not print usage", output.get().startsWith("Usage"));
+ }
+
+ @Test
+ public void testBadArg() {
+ assertTrue(shell.config("--bogus"));
+ assertTrue("Did not print usage", output.get().startsWith("Usage"));
+ }
+
+ @Test
+ public void testToken() {
+ assertTrue(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName()));
+ assertTrue(output.get().contains(ParameterException.class.getCanonicalName()));
+ }
+
+ @Test
+ public void testTokenAndOption() {
+ assertFalse(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName(), "-u", "foo", "-l", "password=foo"));
+ }
+
+ @Test
+ public void testTokenAndOptionAndPassword() {
+ assertTrue(shell.config("--fake", "-tc", PasswordToken.class.getCanonicalName(), "-l", "password=foo", "-p", "bar"));
+ assertTrue(output.get().contains(ParameterException.class.getCanonicalName()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
index 2929b04,0000000..ad24e6d
mode 100644,000000..100644
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellSetInstanceTest.java
@@@ -1,240 -1,0 +1,241 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util.shell;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.powermock.api.easymock.PowerMock.createMock;
+import static org.powermock.api.easymock.PowerMock.expectLastCall;
+import static org.powermock.api.easymock.PowerMock.expectNew;
+import static org.powermock.api.easymock.PowerMock.mockStatic;
+import static org.powermock.api.easymock.PowerMock.replay;
+import static org.powermock.api.easymock.PowerMock.verify;
+
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
++import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigSanityCheck;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Level;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({Shell.class, ZooUtil.class, ConfigSanityCheck.class})
+public class ShellSetInstanceTest {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ @BeforeClass
+ public static void setupClass() {
+ // This is necessary because PowerMock messes with Hadoop's ability to
+ // determine the current user (see security.UserGroupInformation).
+ System.setProperty("HADOOP_USER_NAME", "test");
+ }
+ @AfterClass
+ public static void teardownClass() {
+ System.clearProperty("HADOOP_USER_NAME");
+ }
+
+ private TestOutputStream output;
+ private Shell shell;
+
+ @Before
+ public void setup() throws IOException {
+ Shell.log.setLevel(Level.OFF);
+ output = new TestOutputStream();
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output));
++ shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), output), new PrintWriter(output));
+ shell.setLogErrorsToConsole();
+ }
+ @After
+ public void tearDown() {
+ shell.shutdown();
+ SiteConfiguration.clearInstance();
+ }
+
+ @Test
+ public void testSetInstance_Fake() throws Exception {
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(true);
+ replay(opts);
+ MockInstance theInstance = createMock(MockInstance.class);
+ expectNew(MockInstance.class, "fake").andReturn(theInstance);
+ replay(theInstance, MockInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, MockInstance.class);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_Explicit() throws Exception {
+ testSetInstance_HdfsZooInstance(true, false, false);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_InstanceGiven() throws Exception {
+ testSetInstance_HdfsZooInstance(false, true, false);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_HostsGiven() throws Exception {
+ testSetInstance_HdfsZooInstance(false, false, true);
+ }
+ @Test
+ public void testSetInstance_HdfsZooInstance_Implicit() throws Exception {
+ testSetInstance_HdfsZooInstance(false, false, false);
+ }
+
+ private void testSetInstance_HdfsZooInstance(boolean explicitHdfs, boolean onlyInstance, boolean onlyHosts)
+ throws Exception {
+ ClientConfiguration clientConf = createMock(ClientConfiguration.class);
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(false);
+ expect(opts.getClientConfiguration()).andReturn(clientConf);
+ expect(opts.isHdfsZooInstance()).andReturn(explicitHdfs);
+ if (!explicitHdfs) {
+ expect(opts.getZooKeeperInstance())
+ .andReturn(Collections.<String>emptyList());
+ if (onlyInstance) {
+ expect(opts.getZooKeeperInstanceName()).andReturn("instance");
+ expect(clientConf.withInstance("instance")).andReturn(clientConf);
+ } else {
+ expect(opts.getZooKeeperInstanceName()).andReturn(null);
+ }
+ if (onlyHosts) {
+ expect(opts.getZooKeeperHosts()).andReturn("host3,host4");
+ expect(clientConf.withZkHosts("host3,host4")).andReturn(clientConf);
+ } else {
+ expect(opts.getZooKeeperHosts()).andReturn(null);
+ }
+ }
+ replay(opts);
+
+ if (!onlyInstance) {
+ expect(clientConf.get(ClientProperty.INSTANCE_NAME)).andReturn(null);
+ }
+
+ mockStatic(ConfigSanityCheck.class);
+ ConfigSanityCheck.validate(EasyMock.<AccumuloConfiguration>anyObject());
+ expectLastCall().atLeastOnce();
+ replay(ConfigSanityCheck.class);
+
+ if (!onlyHosts) {
+ expect(clientConf.containsKey(Property.INSTANCE_ZK_HOST.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_ZK_HOST.getKey())).andReturn("host1,host2").atLeastOnce();
+ expect(clientConf.withZkHosts("host1,host2")).andReturn(clientConf);
+ }
+ if (!onlyInstance) {
+ expect(clientConf.containsKey(Property.INSTANCE_VOLUMES.getKey())).andReturn(false).atLeastOnce();
+ expect(clientConf.containsKey(Property.INSTANCE_DFS_DIR.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.containsKey(Property.INSTANCE_DFS_URI.getKey())).andReturn(true).atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_DFS_URI.getKey())).andReturn("hdfs://nn1").atLeastOnce();
+ expect(clientConf.getString(Property.INSTANCE_DFS_DIR.getKey())).andReturn("/dfs").atLeastOnce();
+ }
+
+ UUID randomUUID = null;
+ if (!onlyInstance) {
+ mockStatic(ZooUtil.class);
+ randomUUID = UUID.randomUUID();
+ expect(ZooUtil.getInstanceIDFromHdfs(anyObject(Path.class), anyObject(AccumuloConfiguration.class)))
+ .andReturn(randomUUID.toString());
+ replay(ZooUtil.class);
+ expect(clientConf.withInstance(randomUUID)).andReturn(clientConf);
+ }
+ replay(clientConf);
+
+ ZooKeeperInstance theInstance = createMock(ZooKeeperInstance.class);
+
+ expectNew(ZooKeeperInstance.class, clientConf).andReturn(theInstance);
+ replay(theInstance, ZooKeeperInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, ZooKeeperInstance.class);
+ }
+ @Test
+ public void testSetInstance_ZKInstance_DashZ() throws Exception {
+ testSetInstance_ZKInstance(true);
+ }
+ @Test
+ public void testSetInstance_ZKInstance_DashZIandZH() throws Exception {
+ testSetInstance_ZKInstance(false);
+ }
+ private void testSetInstance_ZKInstance(boolean dashZ) throws Exception {
+ ClientConfiguration clientConf = createMock(ClientConfiguration.class);
+ ShellOptionsJC opts = createMock(ShellOptionsJC.class);
+ expect(opts.isFake()).andReturn(false);
+ expect(opts.getClientConfiguration()).andReturn(clientConf);
+ expect(opts.isHdfsZooInstance()).andReturn(false);
+ if (dashZ) {
+ expect(clientConf.withInstance("foo")).andReturn(clientConf);
+ expect(clientConf.withZkHosts("host1,host2")).andReturn(clientConf);
+ List<String> zl = new java.util.ArrayList<String>();
+ zl.add("foo");
+ zl.add("host1,host2");
+ expect(opts.getZooKeeperInstance()).andReturn(zl);
+ expectLastCall().anyTimes();
+ } else {
+ expect(clientConf.withInstance("bar")).andReturn(clientConf);
+ expect(clientConf.withZkHosts("host3,host4")).andReturn(clientConf);
+ expect(opts.getZooKeeperInstance()).andReturn(Collections.<String>emptyList());
+ expect(opts.getZooKeeperInstanceName()).andReturn("bar");
+ expect(opts.getZooKeeperHosts()).andReturn("host3,host4");
+ }
+ replay(clientConf);
+ replay(opts);
+
+ ZooKeeperInstance theInstance = createMock(ZooKeeperInstance.class);
+ expectNew(ZooKeeperInstance.class, clientConf).andReturn(theInstance);
+ replay(theInstance, ZooKeeperInstance.class);
+
+ shell.setInstance(opts);
+ verify(theInstance, ZooKeeperInstance.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
index 4771f2c,8505370..82d1f34
--- a/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/util/shell/ShellTest.java
@@@ -19,18 -19,16 +19,20 @@@ package org.apache.accumulo.core.util.s
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
+ import java.io.OutputStreamWriter;
+ import java.io.PrintWriter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
+import org.apache.accumulo.core.util.format.DateStringFormatter;
import org.apache.log4j.Level;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@@ -103,8 -82,8 +105,9 @@@ public class ShellTest
public void setup() throws IOException {
Shell.log.setLevel(Level.OFF);
output = new TestOutputStream();
- PrintWriter pw = new PrintWriter( new OutputStreamWriter(output));
- shell = new Shell(new ConsoleReader(new FileInputStream(FileDescriptor.in), new OutputStreamWriter(output)), pw);
+ input = new StringInputStream();
- shell = new Shell(new ConsoleReader(input, output));
++ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
++ shell = new Shell(new ConsoleReader(input, output), pw);
shell.setLogErrorsToConsole();
shell.config("--fake", "-u", "test", "-p", "secret");
}
[15/23] git commit: ACCUMULO-2433 Add a way to still use the exec
methods but provide a custom error message on failure.
Posted by el...@apache.org.
ACCUMULO-2433 Add a way to still use the exec methods but provide a custom error message on failure.
Fix some tests that I've seen fail to provide better messages on failure to help future debugging
because I can't get them reproduced locally in any reasonable timeframe.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4cc25caa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4cc25caa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4cc25caa
Branch: refs/heads/ACCUMULO-2061
Commit: 4cc25caa07ea5ca49e4d81f9e5a22908e45bba59
Parents: 72395e0
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 14:59:09 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 17:13:22 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/test/ShellServerTest.java | 128 ++++++++++++++-----
1 file changed, 99 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4cc25caa/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
index e2ebbcf..d66f509 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
@@ -31,6 +31,10 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -53,6 +57,7 @@ import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
import org.apache.accumulo.server.trace.TraceServer;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -85,7 +90,19 @@ public class ShellServerTest {
}
}
+ private static abstract class ErrorMessageCallback {
+ public abstract String getErrorMessage();
+ }
+
+ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
+ private static final String empty = "";
+ public String getErrorMessage() {
+ return empty;
+ }
+ }
+
private static final Logger log = Logger.getLogger(ShellServerTest.class);
+ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
private static String secret = "superSecret";
public static TemporaryFolder folder = new TemporaryFolder();
@@ -104,36 +121,51 @@ public class ShellServerTest {
}
String exec(String cmd, boolean expectGoodExit) throws IOException {
+ return exec(cmd, expectGoodExit, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit("", true);
+ assertGoodExit("", true, callback);
else
- assertBadExit("", true);
+ assertBadExit("", true, callback);
return result;
}
String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
+ return exec(cmd, expectGoodExit, expectString, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, true, callback);
}
String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
+ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
String result = exec(cmd);
if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
+ assertGoodExit(expectString, stringPresent, callback);
else
- assertBadExit(expectString, stringPresent);
+ assertBadExit(expectString, stringPresent, callback);
return result;
}
- void assertGoodExit(String s, boolean stringPresent) {
+ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
- assertEquals(0, shell.getExitCode());
+ if (0 != shell.getExitCode()) {
+ String errorMsg = callback.getErrorMessage();
+ assertEquals(0, shell.getExitCode());
+ }
if (s.length() > 0)
assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
}
- void assertBadExit(String s, boolean stringPresent) {
+ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
Shell.log.debug(output.get());
assertTrue(shell.getExitCode() > 0);
if (s.length() > 0)
@@ -409,7 +441,16 @@ public class ShellServerTest {
final String table = name.getMethodName();
// addauths
exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
+ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
+ public String getErrorMessage() {
+ try {
+ Connector c = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()).getConnector("root", new PasswordToken(secret));
+ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
+ } catch (Exception e) {
+ return "Could not check authorizations";
+ }
+ }
+ });
exec("addauths -s foo,bar", true);
exec("getauths", true, "foo", true);
exec("getauths", true, "bar", true);
@@ -486,10 +527,10 @@ public class ShellServerTest {
exec("flush -w");
exec("insert n 1 2 3");
exec("flush -w");
- oldCount = countFiles(tableId);
+ List<String> oldFiles = getFiles(tableId);
// at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
+ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
// compact some data:
exec("compact -b g -e z -w");
@@ -564,7 +605,8 @@ public class ShellServerTest {
exec("addsplits row5 row7");
make10();
exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
+ List<String> files = getFiles(tableId);
+ assertEquals("Found the following files: " + files, 3, files.size());
exec("deleterows -t " + table + " -b row5 -e row7", true);
assertEquals(2, countFiles(tableId));
exec("deletetable -f " + table);
@@ -767,12 +809,13 @@ public class ShellServerTest {
final String table = name.getMethodName();
exec("createtable " + table, true);
+
+ // Should be about a 3 second scan
+ for (int i = 0; i < 6; i++) {
+ exec("insert " + i + " cf cq value", true);
+ }
exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
+ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
Thread thread = new Thread() {
public void run() {
try {
@@ -788,14 +831,30 @@ public class ShellServerTest {
}
};
thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
+
+ List<String> scans = new ArrayList<String>();
+ // Try to find the active scan for about 5seconds
+ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
+ String currentScans = exec("listscans", true);
+ String[] lines = currentScans.split("\n");
+ for (int scanOffset = 2; i < lines.length; i++) {
+ String currentScan = lines[scanOffset];
+ if (currentScan.contains(table)) {
+ scans.add(currentScan);
+ }
+ }
+ UtilWaitThread.sleep(100);
+ }
thread.join();
+
+ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
+
+ for (String scan : scans) {
+ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
+ String parts[] = scan.split("\\|");
+ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
+ }
+
exec("deletetable -f " + table, true);
}
@@ -873,13 +932,24 @@ public class ShellServerTest {
exec(String.format("insert row%d cf col%d value", i, i));
}
}
+
+ private List<String> getFiles(String tableId) throws IOException {
+ output.clear();
- private int countFiles(String tableId) throws IOException {
exec("scan -t !METADATA -np -c file -b " + tableId + " -e " + tableId + "~");
-
- log.debug("countFiles(): " + output.get());
-
- return output.get().split("\n").length - 1;
+
+ String[] lines = StringUtils.split(output.get(), "\n");
+ output.clear();
+
+ if (0 == lines.length) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
+ }
+
+ private int countFiles(String tableId) throws IOException {
+ return getFiles(tableId).size();
}
private String getTableId(String tableName) throws Exception {
[10/23] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.4.5-SNAPSHOT' into 1.5.2-SNAPSHOT
-sours merge, 1.5.x specific changes in follow on patch.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/32dc1040
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/32dc1040
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/32dc1040
Branch: refs/heads/ACCUMULO-2061
Commit: 32dc1040e7914a5772a529091a1e2f061f637c25
Parents: 12b53b0 6faac42
Author: Sean Busbey <bu...@cloudera.com>
Authored: Mon Mar 10 09:22:11 2014 -0500
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Mon Mar 10 09:22:11 2014 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[08/23] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Conflicts:
server/src/main/java/org/apache/accumulo/server/master/Master.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a63e7c9b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a63e7c9b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a63e7c9b
Branch: refs/heads/ACCUMULO-2061
Commit: a63e7c9b9767c0585956616597385f32463be688
Parents: 0297276 12b53b0
Author: Josh Elser <el...@apache.org>
Authored: Sun Mar 9 23:49:47 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sun Mar 9 23:49:47 2014 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[03/23] git commit: ACCUMULO-2061 Fix FileUtil
Posted by el...@apache.org.
ACCUMULO-2061 Fix FileUtil
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3b3ccbb0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3b3ccbb0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3b3ccbb0
Branch: refs/heads/ACCUMULO-2061
Commit: 3b3ccbb02e3d0e2cb5f46d506ab1080d6f95d2d6
Parents: 0c84156
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 7 16:46:47 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 7 16:46:47 2014 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/util/FileUtil.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3b3ccbb0/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index 8e38cbd..277a0c2 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -125,7 +125,7 @@ public class FileUtil {
String newMapFile = String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION);
outFiles.add(newMapFile);
- FileSystem ns = fs.getFileSystemByPath(new Path(newMapFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(newMapFile)).getFileSystem();
FileSKVWriter writer = new RFileOperations().openWriter(newMapFile.toString(), ns, ns.getConf(), acuConf);
writer.startDefaultLocalityGroup();
List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<SortedKeyValueIterator<Key,Value>>(inFiles.size());
@@ -133,7 +133,7 @@ public class FileUtil {
FileSKVIterator reader = null;
try {
for (String s : inFiles) {
- ns = fs.getFileSystemByPath(new Path(s));
+ ns = fs.getVolumeByPath(new Path(s)).getFileSystem();
reader = FileOperations.getInstance().openIndex(s, ns, ns.getConf(), acuConf);
iters.add(reader);
}
@@ -390,7 +390,7 @@ public class FileUtil {
for (String ref : mapFiles) {
FileSKVIterator reader = null;
Path path = new Path(ref);
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
try {
if (useIndex)
reader = FileOperations.getInstance().openIndex(path.toString(), ns, ns.getConf(), acuConf);
@@ -435,7 +435,7 @@ public class FileUtil {
for (FileRef mapfile : mapfiles) {
FileSKVIterator reader = null;
- FileSystem ns = fs.getFileSystemByPath(mapfile.path());
+ FileSystem ns = fs.getVolumeByPath(mapfile.path()).getFileSystem();
try {
reader = FileOperations.getInstance().openReader(mapfile.toString(), false, ns, ns.getConf(), acuConf);
@@ -470,7 +470,7 @@ public class FileUtil {
for (FileRef ref : mapFiles) {
Path path = ref.path();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), acuConf);
try {
@@ -513,7 +513,7 @@ public class FileUtil {
counts.put(keyExtent, new MLong(0));
Text row = new Text();
- FileSystem ns = fs.getFileSystemByPath(mapFile);
+ FileSystem ns = fs.getVolumeByPath(mapFile).getFileSystem();
FileSKVIterator index = FileOperations.getInstance().openIndex(mapFile.toString(), ns, ns.getConf(), acuConf);
try {
[04/23] git commit: ACCUMULO-2061 Fix the rest of server-base
Posted by el...@apache.org.
ACCUMULO-2061 Fix the rest of server-base
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e613f710
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e613f710
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e613f710
Branch: refs/heads/ACCUMULO-2061
Commit: e613f7105e3a0828096909535147b4bc80a265f6
Parents: 3b3ccbb
Author: Josh Elser <el...@apache.org>
Authored: Fri Mar 7 17:14:17 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Mar 7 17:14:17 2014 -0500
----------------------------------------------------------------------
.../apache/accumulo/server/client/BulkImporter.java | 4 ++--
.../accumulo/server/client/HdfsZooInstance.java | 15 ++++++++++++++-
.../accumulo/server/conf/ZooConfiguration.java | 16 +++++++++++++++-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 10 ++--------
.../org/apache/accumulo/server/init/Initialize.java | 2 +-
.../server/master/recovery/HadoopLogCloser.java | 2 +-
.../server/master/recovery/MapRLogCloser.java | 2 +-
.../apache/accumulo/server/util/LocalityCheck.java | 2 +-
.../accumulo/server/util/TabletOperations.java | 4 ++--
9 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
index dc9acf8..27ab078 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/BulkImporter.java
@@ -339,7 +339,7 @@ public class BulkImporter {
try {
for (Path path : paths) {
- FileSystem fs = vm.getFileSystemByPath(path);
+ FileSystem fs = vm.getVolumeByPath(path).getFileSystem();
mapFileSizes.put(path, fs.getContentSummary(path).getLength());
}
} catch (IOException e) {
@@ -639,7 +639,7 @@ public class BulkImporter {
Collection<ByteSequence> columnFamilies = Collections.emptyList();
String filename = file.toString();
// log.debug(filename + " finding overlapping tablets " + startRow + " -> " + endRow);
- FileSystem fs = vm.getFileSystemByPath(file);
+ FileSystem fs = vm.getVolumeByPath(file).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(filename, true, fs, fs.getConf(), acuConf);
try {
Text row = startRow;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 6993a0a..8aa0be9 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.client;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -38,10 +39,13 @@ import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
@@ -122,7 +126,16 @@ public class HdfsZooInstance implements Instance {
private static synchronized void _getInstanceID() {
if (instanceId == null) {
- String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), ServerConfiguration.getSiteConfiguration());
+ AccumuloConfiguration acuConf = ServerConfiguration.getSiteConfiguration();
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
instanceId = instanceIdFromFile;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
index 32f6126..94e468b 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/ZooConfiguration.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.conf;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -25,10 +26,15 @@ import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance.AccumuloNotInitializedException;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.log4j.Logger;
public class ZooConfiguration extends AccumuloConfiguration {
@@ -57,7 +63,15 @@ public class ZooConfiguration extends AccumuloConfiguration {
if (instance == null) {
propCache = new ZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT));
instance = new ZooConfiguration(parent);
- String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(), parent);
+ // InstanceID should be the same across all volumes, so just choose one
+ VolumeManager fs;
+ try {
+ fs = VolumeManagerImpl.get();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Volume randVolume = fs.getVolumes().iterator().next();
+ String deprecatedInstanceIdFromHdfs = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), parent);
instanceId = deprecatedInstanceIdFromHdfs;
}
return instance;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 42ea912..110f1e4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -26,21 +26,17 @@ import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -49,8 +45,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
-import com.google.common.base.Preconditions;
-
/**
* Utility methods for managing absolute URIs contained in Accumulo metadata.
*/
@@ -245,8 +239,8 @@ public class VolumeUtil {
// this code needs to be idempotent
- FileSystem fs1 = vm.getFileSystemByPath(dir);
- FileSystem fs2 = vm.getFileSystemByPath(newDir);
+ FileSystem fs1 = vm.getVolumeByPath(dir).getFileSystem();
+ FileSystem fs2 = vm.getVolumeByPath(newDir).getFileSystem();
if (!same(fs1, dir, fs2, newDir)) {
if (fs2.exists(newDir)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 59fad73..206a721 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -341,7 +341,7 @@ public class Initialize {
// the root tablet contains the key extent and locations of all the
// metadata tablets
String initRootTabFile = rootTablet + "/00000_00000." + FileOperations.getNewFileExtension(AccumuloConfiguration.getDefaultConfiguration());
- FileSystem ns = fs.getFileSystemByPath(new Path(initRootTabFile));
+ FileSystem ns = fs.getVolumeByPath(new Path(initRootTabFile)).getFileSystem();
FileSKVWriter mfw = FileOperations.getInstance().openWriter(initRootTabFile, ns, ns.getConf(), AccumuloConfiguration.getDefaultConfiguration());
mfw.startDefaultLocalityGroup();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
index 7edc0cf..0e7f83a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java
@@ -37,7 +37,7 @@ public class HadoopLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path source) throws IOException {
- FileSystem ns = fs.getFileSystemByPath(source);
+ FileSystem ns = fs.getVolumeByPath(source).getFileSystem();
// if path points to a viewfs path, then resolve to underlying filesystem
if (ViewFSUtils.isViewFS(ns)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
index bba7ac5..d737109 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/recovery/MapRLogCloser.java
@@ -32,7 +32,7 @@ public class MapRLogCloser implements LogCloser {
@Override
public long close(AccumuloConfiguration conf, VolumeManager fs, Path path) throws IOException {
log.info("Recovering file " + path.toString() + " by changing permission to readonly");
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FsPermission roPerm = new FsPermission((short) 0444);
try {
ns.setPermission(path, roPerm);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
index a96e791..35bb8f5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/LocalityCheck.java
@@ -88,7 +88,7 @@ public class LocalityCheck {
}
for (String file : files) {
Path filePath = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(filePath);
+ FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem();
FileStatus fileStatus = ns.getFileStatus(filePath);
BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation blockLocation : fileBlockLocations) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e613f710/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
index b237cd0..b8e7113 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
@@ -46,7 +46,7 @@ public class TabletOperations {
lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
- FileSystem pathFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory()).toString();
}
log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
@@ -56,7 +56,7 @@ public class TabletOperations {
if (fs.exists(lowDirectoryPath))
throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
if (fs.mkdirs(lowDirectoryPath)) {
- FileSystem lowDirectoryFs = fs.getFileSystemByPath(lowDirectoryPath);
+ FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
return lowDirectoryPath.makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory()).toString();
}
}
[06/23] git commit: ACCUMULO-2061 Get all of the unit tests running
successfully
Posted by el...@apache.org.
ACCUMULO-2061 Get all of the unit tests running successfully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e97b093b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e97b093b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e97b093b
Branch: refs/heads/ACCUMULO-2061
Commit: e97b093b320eb5025d619e7fa1494b4c150599b8
Parents: e613f71
Author: Josh Elser <el...@apache.org>
Authored: Sat Mar 8 00:34:53 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Sat Mar 8 00:34:53 2014 -0500
----------------------------------------------------------------------
.../core/volume/VolumeConfiguration.java | 3 ++-
.../accumulo/server/fs/VolumeManagerImpl.java | 4 +--
.../accumulo/master/tableOps/ExportTable.java | 2 +-
.../accumulo/master/tableOps/ImportTable.java | 2 +-
.../monitor/servlets/DefaultServlet.java | 2 +-
.../tserver/BulkFailedCopyProcessor.java | 27 ++++++++++----------
.../org/apache/accumulo/tserver/Compactor.java | 4 +--
.../apache/accumulo/tserver/FileManager.java | 2 +-
.../org/apache/accumulo/tserver/Tablet.java | 4 +--
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../compaction/MajorCompactionRequest.java | 2 +-
.../apache/accumulo/tserver/log/LogSorter.java | 2 +-
.../accumulo/tserver/log/MultiReader.java | 2 +-
.../apache/accumulo/tserver/RootFilesTest.java | 1 +
.../tserver/TabletServerSyncCheckTest.java | 12 +++++----
.../accumulo/tserver/log/MultiReaderTest.java | 8 +++---
.../tserver/log/SortedLogRecoveryTest.java | 9 ++++---
.../tserver/log/TestUpgradePathForWALogs.java | 6 ++---
.../performance/scan/CollectTabletStats.java | 6 ++---
.../accumulo/test/functional/BulkFileIT.java | 2 +-
20 files changed, 54 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 460ffbe..3f4ab3e 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -128,7 +128,8 @@ public class VolumeConfiguration {
*/
@SuppressWarnings("deprecation")
public static <T extends FileSystem> Volume create(T fs, AccumuloConfiguration acuconf) {
- return new VolumeImpl(fs, acuconf.get(Property.INSTANCE_DFS_DIR));
+ String dfsDir = acuconf.get(Property.INSTANCE_DFS_DIR);
+ return new VolumeImpl(fs, null == dfsDir ? Property.INSTANCE_DFS_DIR.getDefaultValue() : dfsDir);
}
public static <T extends FileSystem> Volume create(T fs, String basePath) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index e19182d..a32b0bd 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -87,9 +87,9 @@ public class VolumeManagerImpl implements VolumeManager {
}
}
- public static org.apache.accumulo.server.fs.VolumeManager getLocal() throws IOException {
+ public static org.apache.accumulo.server.fs.VolumeManager getLocal(String localBasePath) throws IOException {
AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
- Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), accConf);
+ Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
return new VolumeManagerImpl(Collections.singletonMap("", defaultLocalVolume), "", accConf);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
index 36bbb53..eeb9b16 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java
@@ -152,7 +152,7 @@ class WriteExportFiles extends MasterRepo {
public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception {
fs.mkdirs(new Path(exportDir));
- Path exportMetaFilePath = fs.getFileSystemByPath(new Path(exportDir)).makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
+ Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE));
FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false);
ZipOutputStream zipOut = new ZipOutputStream(fileOut);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
index 7e84c55..cd59b78 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java
@@ -427,7 +427,7 @@ class ImportPopulateZookeeper extends MasterRepo {
Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE);
try {
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
return TableOperationsImpl.getExportedProps(ns, path);
} catch (IOException ioe) {
throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
----------------------------------------------------------------------
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
index 0da06cb..31e63ed 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java
@@ -270,7 +270,7 @@ public class DefaultServlet extends BasicServlet {
try {
for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) {
final Path basePath = new Path(baseDir);
- final FileSystem fs = vm.getFileSystemByPath(basePath);
+ final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem();
try {
// Calculate the amount of space used by Accumulo on the FileSystem
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
index 5c64970..6b31af1 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.volume.VolumeConfiguration;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
@@ -32,35 +33,35 @@ import org.apache.log4j.Logger;
* Copy failed bulk imports.
*/
public class BulkFailedCopyProcessor implements Processor {
-
+
private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class);
-
+
@Override
public Processor newProcessor() {
return new BulkFailedCopyProcessor();
}
-
+
@Override
public void process(String workID, byte[] data) {
-
+
String paths[] = new String(data, Constants.UTF8).split(",");
-
+
Path orig = new Path(paths[0]);
Path dest = new Path(paths[1]);
Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp");
-
+
try {
- FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.volume.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(),
- ServerConfiguration.getSiteConfiguration()));
-
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(),
+ ServerConfiguration.getSiteConfiguration()).getFileSystem());
+
FileUtil.copy(fs, orig, fs, tmp, false, true, CachedConfiguration.getInstance());
fs.rename(tmp, dest);
log.debug("copied " + orig + " to " + dest);
} catch (IOException ex) {
try {
- FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.volume.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(),
- ServerConfiguration.getSiteConfiguration()));
-
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(),
+ ServerConfiguration.getSiteConfiguration()).getFileSystem());
+
fs.create(dest).close();
log.warn(" marked " + dest + " failed", ex);
} catch (IOException e) {
@@ -69,5 +70,5 @@ public class BulkFailedCopyProcessor implements Processor {
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
index 151db6e..822171c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
@@ -336,7 +336,7 @@ public class Compactor implements Callable<CompactionStats> {
thread = Thread.currentThread();
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = this.fs.getFileSystemByPath(outputFile.path());
+ FileSystem ns = this.fs.getVolumeByPath(outputFile.path()).getFileSystem();
mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf);
Map<String,Set<ByteSequence>> lGroups;
@@ -421,7 +421,7 @@ public class Compactor implements Callable<CompactionStats> {
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem fs = this.fs.getFileSystemByPath(mapFile.path());
+ FileSystem fs = this.fs.getVolumeByPath(mapFile.path()).getFileSystem();
FileSKVIterator reader;
reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
index bb95532..8bf2517 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
@@ -309,7 +309,7 @@ public class FileManager {
if (!file.contains(":"))
throw new IllegalArgumentException("Expected uri, got : " + file);
Path path = new Path(file);
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
//log.debug("Opening "+file + " path " + path);
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()),
dataCache, indexCache);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
index cc4b68d..3fe60b7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java
@@ -1260,7 +1260,7 @@ public class Tablet {
long rtime = Long.MIN_VALUE;
for (FileRef ref : datafiles.keySet()) {
Path path = ref.path();
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent));
long maxTime = -1;
try {
@@ -2975,7 +2975,7 @@ public class Tablet {
FileOperations fileFactory = FileOperations.getInstance();
for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) {
FileRef file = entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration());
try {
Key first = openReader.getFirstKey();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 475621b..6d73125 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -969,7 +969,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
Path path = new Path(mapping.getKey());
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
path = ns.makeQualified(path);
fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
index 3bbb476..900600f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
@@ -81,7 +81,7 @@ public class MajorCompactionRequest implements Cloneable {
// @TODO verify the file isn't some random file in HDFS
// @TODO ensure these files are always closed?
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = volumeManager.getFileSystemByPath(ref.path());
+ FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem();
FileSKVIterator openReader = fileFactory.openReader(ref.path().toString(), true, ns, ns.getConf(), tableConfig);
return openReader;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index 8f783c3..bb8e3c7 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -160,7 +160,7 @@ public class LogSorter {
private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException {
Path path = new Path(destPath, String.format("part-r-%05d", part++));
- FileSystem ns = fs.getFileSystemByPath(path);
+ FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
@SuppressWarnings("deprecation")
MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
index a28bac4..541f075 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java
@@ -97,7 +97,7 @@ public class MultiReader {
foundFinish = true;
continue;
}
- FileSystem ns = fs.getFileSystemByPath(child.getPath());
+ FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem();
heap.add(new Index(new Reader(ns, child.getPath().toString(), ns.getConf())));
}
if (!foundFinish)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
index 1cd8f12..7a1c84b 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java
@@ -116,6 +116,7 @@ public class RootFilesTest {
ConfigurationCopy conf = new ConfigurationCopy();
conf.set(Property.INSTANCE_DFS_URI, "file:///");
+ conf.set(Property.INSTANCE_DFS_DIR, "/");
VolumeManager vm = VolumeManagerImpl.get(conf);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
index 50c8b31..a5a1deb 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
@@ -22,6 +22,8 @@ import java.util.Map;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.volume.Volume;
+import org.apache.accumulo.core.volume.VolumeImpl;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -44,7 +46,7 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_DURABLE_SYNC, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/")));
vm.ensureSyncIsEnabled();
}
@@ -56,7 +58,7 @@ public class TabletServerSyncCheckTest {
FileSystem fs1 = new TestFileSystem(conf1);
FileSystem fs2 = new TestFileSystem(conf2);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("bar", fs2, "foo", fs1));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("bar", new VolumeImpl(fs2, "/"), "foo", new VolumeImpl(fs1, "/")));
vm.ensureSyncIsEnabled();
}
@@ -67,7 +69,7 @@ public class TabletServerSyncCheckTest {
conf.set(DFS_SUPPORT_APPEND, "false");
FileSystem fs = new TestFileSystem(conf);
- TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs));
+ TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/")));
vm.ensureSyncIsEnabled();
}
@@ -89,7 +91,7 @@ public class TabletServerSyncCheckTest {
private class TestVolumeManagerImpl extends VolumeManagerImpl {
- public TestVolumeManagerImpl(Map<String,? extends FileSystem> volumes) {
+ public TestVolumeManagerImpl(Map<String,Volume> volumes) {
super(volumes, volumes.keySet().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
}
@@ -149,7 +151,7 @@ public class TabletServerSyncCheckTest {
}
@Override
- public FileSystem getFileSystemByPath(Path path) {
+ public Volume getVolumeByPath(Path path) {
return null;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
index c4d3dfb..a79e77e 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java
@@ -42,13 +42,13 @@ public class MultiReaderTest {
@Before
public void setUp() throws Exception {
- fs = VolumeManagerImpl.getLocal();
root.create();
- String path = root.getRoot().getAbsolutePath();
- Path root = new Path("file://" + path + "/manyMaps");
+ String path = root.getRoot().getAbsolutePath() + "/manyMaps";
+ fs = VolumeManagerImpl.getLocal(path);
+ Path root = new Path("file://" + path);
fs.mkdirs(root);
fs.create(new Path(root, "finished")).close();
- FileSystem ns = fs.getFileSystemByPath(root);
+ FileSystem ns = fs.getVolumeByPath(root).getFileSystem();
@SuppressWarnings("deprecation")
Writer oddWriter = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 359bfa1..fffa15e 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@ -117,14 +117,15 @@ public class SortedLogRecoveryTest {
private static List<Mutation> recover(Map<String,KeyValue[]> logs, Set<String> files, KeyExtent extent) throws IOException {
TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
root.create();
- final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir";
- VolumeManager fs = VolumeManagerImpl.getLocal();
- fs.deleteRecursively(new Path(workdir));
+ final String workdir = root.getRoot().getAbsolutePath() + "/workdir";
+ VolumeManager fs = VolumeManagerImpl.getLocal(workdir);
+ final Path workdirPath = new Path("file://" + workdir);
+ fs.deleteRecursively(workdirPath);
ArrayList<Path> dirs = new ArrayList<Path>();
try {
for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
String path = workdir + "/" + entry.getKey();
- FileSystem ns = fs.getFileSystemByPath(new Path(path));
+ FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem();
@SuppressWarnings("deprecation")
Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class);
for (KeyValue lfe : entry.getValue()) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
index af149fa..d6c23e3 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
@@ -61,10 +61,10 @@ public class TestUpgradePathForWALogs {
public void setUp() throws Exception {
// quiet log messages about compress.CodecPool
Logger.getRootLogger().setLevel(Level.ERROR);
- fs = VolumeManagerImpl.getLocal();
root.create();
- String path = root.getRoot().getAbsolutePath();
- Path manyMapsPath = new Path("file://" + path + "/manyMaps");
+ String path = root.getRoot().getAbsolutePath() + "/manyMaps";
+ fs = VolumeManagerImpl.getLocal(path);
+ Path manyMapsPath = new Path("file://" + path);
fs.mkdirs(manyMapsPath);
fs.create(new Path(manyMapsPath, "finished")).close();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
index 9a9cad7..d2c8d0f 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
@@ -398,7 +398,7 @@ public class CollectTabletStats {
// assume it is a map file
status = fs.getFileStatus(new Path(file + "/data"));
}
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen());
System.out.println("\t\t\tBlocks for : " + file);
@@ -445,7 +445,7 @@ public class CollectTabletStats {
HashSet<ByteSequence> columnSet = createColumnBSS(columns);
for (FileRef file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf);
Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true);
reader.seek(range, columnSet, columnSet.size() == 0 ? false : true);
@@ -475,7 +475,7 @@ public class CollectTabletStats {
List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>(files.size());
for (FileRef file : files) {
- FileSystem ns = fs.getFileSystemByPath(file.path());
+ FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem();
readers.add(FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf.getConfiguration()));
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e97b093b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index 5df7b68..eee093b 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@ -53,7 +53,7 @@ public class BulkFileIT extends SimpleMacIT {
c.tableOperations().addSplits(tableName, splits);
Configuration conf = new Configuration();
AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
- FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultFilesystem(conf, aconf));
+ FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getTableNames(1)[0];
[19/23] git commit: Merge remote-tracking branch
'origin/1.6.0-SNAPSHOT' into ACCUMULO-2061
Posted by el...@apache.org.
Merge remote-tracking branch 'origin/1.6.0-SNAPSHOT' into ACCUMULO-2061
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3eb1593a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3eb1593a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3eb1593a
Branch: refs/heads/ACCUMULO-2061
Commit: 3eb1593a450e89851a518aa5dccd454753442824
Parents: e97b093 da7f937
Author: Josh Elser <el...@apache.org>
Authored: Mon Mar 10 18:31:37 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon Mar 10 18:31:37 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/util/shell/Shell.java | 20 +--
.../core/util/shell/ShellConfigTest.java | 5 +-
.../core/util/shell/ShellSetInstanceTest.java | 3 +-
.../accumulo/core/util/shell/ShellTest.java | 5 +-
.../chapters/troubleshooting.tex | 29 ++++
.../apache/accumulo/proxy/SimpleProxyIT.java | 98 +++++++++----
.../org/apache/accumulo/test/DeleteRowsIT.java | 42 ++++++
.../org/apache/accumulo/test/ShellServerIT.java | 145 ++++++++++++++-----
.../apache/accumulo/test/TestDeleteRowsIT.java | 42 ------
9 files changed, 267 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3eb1593a/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
----------------------------------------------------------------------
[12/23] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Delete conflicts; functional tests moved to ITs.
Conflicts:
test/src/main/java/org/apache/accumulo/test/functional/RunTests.java
test/system/auto/README
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4223d668
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4223d668
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4223d668
Branch: refs/heads/ACCUMULO-2061
Commit: 4223d668197a911d0dcc01ca25d54790c0082bcc
Parents: a63e7c9 bc9cee0
Author: Sean Busbey <bu...@cloudera.com>
Authored: Mon Mar 10 10:28:12 2014 -0500
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Mon Mar 10 10:28:12 2014 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[16/23] Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/da7f937b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index 7a1121f,0000000..e0ad1bd
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@@ -1,1159 -1,0 +1,1238 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
++import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
++import java.util.ArrayList;
++import java.util.Arrays;
++import java.util.Collections;
++import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.Namespaces;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.util.shell.Shell;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.apache.accumulo.tracer.TraceServer;
+import org.apache.commons.io.FileUtils;
++import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.tools.DistCp;
+import org.apache.log4j.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class ShellServerIT extends SimpleMacIT {
+ public static class TestOutputStream extends OutputStream {
+ StringBuilder sb = new StringBuilder();
+
+ @Override
+ public void write(int b) throws IOException {
+ sb.append((char) (0xff & b));
+ }
+
+ public String get() {
+ return sb.toString();
+ }
+
+ public void clear() {
+ sb.setLength(0);
+ }
+ }
+
+ private static final Logger log = Logger.getLogger(ShellServerIT.class);
+
+ public static class StringInputStream extends InputStream {
+ private String source = "";
+ private int offset = 0;
+
+ @Override
+ public int read() throws IOException {
+ if (offset == source.length())
+ return '\n';
+ else
+ return source.charAt(offset++);
+ }
+
+ public void set(String other) {
+ source = other;
+ offset = 0;
+ }
+ }
+
++ private static abstract class ErrorMessageCallback {
++ public abstract String getErrorMessage();
++ }
++
++ private static class NoOpErrorMessageCallback extends ErrorMessageCallback {
++ private static final String empty = "";
++ public String getErrorMessage() {
++ return empty;
++ }
++ }
++
++ private static final NoOpErrorMessageCallback noop = new NoOpErrorMessageCallback();
++
+ public TestOutputStream output;
+ public StringInputStream input;
+ public Shell shell;
+ private static Process traceProcess;
+
+ @Rule
+ public TestName name = new TestName();
+
+ String exec(String cmd) throws IOException {
+ output.clear();
+ shell.execCommand(cmd, true, true);
+ return output.get();
+ }
+
+ String exec(String cmd, boolean expectGoodExit) throws IOException {
++ return exec(cmd, expectGoodExit, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
- assertGoodExit("", true);
++ assertGoodExit("", true, callback);
+ else
- assertBadExit("", true);
++ assertBadExit("", true, callback);
+ return result;
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString) throws IOException {
- return exec(cmd, expectGoodExit, expectString, true);
++ return exec(cmd, expectGoodExit, expectString, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, String expectString, ErrorMessageCallback callback) throws IOException {
++ return exec(cmd, expectGoodExit, expectString, true, callback);
+ }
+
+ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent) throws IOException {
++ return exec(cmd, expectGoodExit, expectString, stringPresent, noop);
++ }
++
++ String exec(String cmd, boolean expectGoodExit, String expectString, boolean stringPresent, ErrorMessageCallback callback) throws IOException {
+ String result = exec(cmd);
+ if (expectGoodExit)
- assertGoodExit(expectString, stringPresent);
++ assertGoodExit(expectString, stringPresent, callback);
+ else
- assertBadExit(expectString, stringPresent);
++ assertBadExit(expectString, stringPresent, callback);
+ return result;
+ }
+
+ void assertGoodExit(String s, boolean stringPresent) {
++ assertGoodExit(s, stringPresent, noop);
++ }
++
++ void assertGoodExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.info(output.get());
- assertEquals(0, shell.getExitCode());
++ if (0 != shell.getExitCode()) {
++ String errorMsg = callback.getErrorMessage();
++ assertEquals(errorMsg, 0, shell.getExitCode());
++ }
+
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ }
+
- void assertBadExit(String s, boolean stringPresent) {
++ void assertBadExit(String s, boolean stringPresent, ErrorMessageCallback callback) {
+ Shell.log.debug(output.get());
+ assertTrue(shell.getExitCode() > 0);
+ if (s.length() > 0)
+ assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent, output.get().contains(s));
+ shell.resetExitCode();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ // history file is updated in $HOME
+ System.setProperty("HOME", getFolder().getAbsolutePath());
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
+ // use reflection to call this method so it does not need to be made public
+ Method method = getStaticCluster().getClass().getDeclaredMethod("exec", Class.class, String[].class);
+ method.setAccessible(true);
+ traceProcess = (Process) method.invoke(getStaticCluster(), TraceServer.class, new String[0]);
+
+ // give the tracer some time to start
+ UtilWaitThread.sleep(1000);
+ }
+
+ @Before
+ public void setupShell() throws Exception {
+ // start the shell
+ output = new TestOutputStream();
+ input = new StringInputStream();
- shell = new Shell(new ConsoleReader(input, output));
++ PrintWriter pw = new PrintWriter(new OutputStreamWriter(output));
++ shell = new Shell(new ConsoleReader(input, output), pw);
+ shell.setLogErrorsToConsole();
+ shell.config("-u", "root", "-p", ROOT_PASSWORD, "-z", getStaticCluster().getConfig().getInstanceName(), getStaticCluster().getConfig().getZooKeepers(),
+ "--config-file", getStaticCluster().getConfig().getClientConfFile().getAbsolutePath());
+ exec("quit", true);
+ shell.start();
+ shell.setExit(false);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ traceProcess.destroy();
+ }
+
+ @After
+ public void deleteTables() throws Exception {
+ Connector c = getConnector();
+ for (String table : c.tableOperations().list()) {
+ if (!table.equals(MetadataTable.NAME) && !table.equals(RootTable.NAME) && !table.equals("trace"))
+ try {
+ c.tableOperations().delete(table);
+ } catch (TableNotFoundException e) {
+ // don't care
+ }
+ }
+ }
+
+ @After
+ public void tearDownShell() {
+ shell.shutdown();
+ }
+
+ @Test(timeout = 60000)
+ public void exporttableImporttable() throws Exception {
+ final String table = name.getMethodName(), table2 = table + "2";
+
+ // exporttable / importtable
+ exec("createtable " + table + " -evc", true);
+ make10();
+ exec("addsplits row5", true);
+ exec("config -t " + table + " -s table.split.threshold=345M", true);
+ exec("offline " + table, true);
+ String export = "file://" + new File(getFolder(), "ShellServerIT.export").toString();
+ exec("exporttable -t " + table + " " + export, true);
+ DistCp cp = newDistCp();
+ String import_ = "file://" + new File(getFolder(), "ShellServerIT.import").toString();
+ cp.run(new String[] {"-f", export + "/distcp.txt", import_});
+ exec("importtable " + table2 + " " + import_, true);
+ exec("config -t " + table2 + " -np", true, "345M", true);
+ exec("getsplits -t " + table2, true, "row5", true);
+ exec("constraint --list -t " + table2, true, "VisibilityConstraint=2", true);
+ exec("onlinetable " + table, true);
+ exec("deletetable -f " + table, true);
+ exec("deletetable -f " + table2, true);
+ }
+
+ private DistCp newDistCp() {
+ try {
+ @SuppressWarnings("unchecked")
+ Constructor<DistCp>[] constructors = (Constructor<DistCp>[]) DistCp.class.getConstructors();
+ for (Constructor<DistCp> constructor : constructors) {
+ Class<?>[] parameterTypes = constructor.getParameterTypes();
+ if (parameterTypes.length > 0 && parameterTypes[0].equals(Configuration.class)) {
+ if (parameterTypes.length == 1) {
+ return constructor.newInstance(new Configuration());
+ } else if (parameterTypes.length == 2) {
+ return constructor.newInstance(new Configuration(), null);
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ throw new RuntimeException("Unexpected constructors for DistCp");
+ }
+
+ @Test(timeout = 45000)
+ public void setscaniterDeletescaniter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setscaniter, deletescaniter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING");
+ exec("setscaniter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("scan", true, "3", true);
+ exec("deletescaniter -n name", true);
+ exec("scan", true, "1", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void execfile() throws Exception {
+ // execfile
+ File file = File.createTempFile("ShellServerIT.execfile", ".conf", getFolder());
+ PrintWriter writer = new PrintWriter(file.getAbsolutePath());
+ writer.println("about");
+ writer.close();
+ exec("execfile " + file.getAbsolutePath(), true, Constants.VERSION, true);
+
+ }
+
+ @Test(timeout = 45000)
+ public void egrep() throws Exception {
+ final String table = name.getMethodName();
+
+ // egrep
+ exec("createtable " + table);
+ make10();
+ String lines = exec("egrep row[123]", true);
+ assertTrue(lines.split("\n").length - 1 == 3);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void du() throws Exception {
+ final String table = name.getMethodName();
+
+ // create and delete a table so we get out of a table context in the shell
+ exec("notable", true);
+
+ // Calling du not in a table context shouldn't throw an error
+ output.clear();
+ exec("du", true, "", true);
+
+ output.clear();
+ exec("createtable " + table);
+ make10();
+ exec("flush -t " + table + " -w");
+ exec("du " + table, true, " [" + table + "]", true);
+ output.clear();
+ shell.execCommand("du -h", false, false);
+ String o = output.get();
+ // for some reason, there's a bit of fluctuation
+ assertTrue("Output did not match regex: '" + o + "'", o.matches(".*[1-9][0-9][0-9]\\s\\[" + table + "\\]\\n"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 1000)
+ public void debug() throws Exception {
+ exec("debug", true, "off", true);
+ exec("debug on", true);
+ exec("debug", true, "on", true);
+ exec("debug off", true);
+ exec("debug", true, "off", true);
+ exec("debug debug", false);
+ exec("debug debug debug", false);
+ }
+
+ @Test(timeout = 45000)
+ public void user() throws Exception {
+ final String table = name.getMethodName();
+
+ // createuser, deleteuser, user, users, droptable, grant, revoke
+ input.set("secret\nsecret\n");
+ exec("createuser xyzzy", true);
+ exec("users", true, "xyzzy", true);
+ String perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains("Table permissions (" + MetadataTable.NAME + "): Table.READ"));
+ exec("grant -u xyzzy -s System.CREATE_TABLE", true);
+ perms = exec("userpermissions -u xyzzy", true);
+ assertTrue(perms.contains(""));
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("grant -u root -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("grant -u root -s foo", false);
+ exec("grant -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ input.set("secret\nsecret\n");
+ exec("user xyzzy", true);
+ exec("createtable " + table, true, "xyzzy@", true);
+ exec("insert row1 cf cq 1", true);
+ exec("scan", true, "row1", true);
+ exec("droptable -f " + table, true);
+ exec("deleteuser xyzzy", false, "delete yourself", true);
+ input.set(ROOT_PASSWORD + "\n" + ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ exec("revoke -u xyzzy -s System.CREATE_TABLE", true);
+ exec("revoke -u xyzzy -s System.GOOFY", false);
+ exec("revoke -u xyzzy -s foo", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.WRITE", true);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " Table.GOOFY", false);
+ exec("revoke -u xyzzy -t " + MetadataTable.NAME + " foo", false);
+ exec("deleteuser xyzzy", true);
+ exec("users", true, "xyzzy", false);
+ }
+
+ @Test(timeout = 45000)
+ public void iter() throws Exception {
+ final String table = name.getMethodName();
+
+ // setshelliter, listshelliter, deleteshelliter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n name", true);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n name", false);
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -pn sum -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setshelliter -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -pn sum -n xyzzy", true);
+ exec("scan -pn sum", true, "3", true);
+ exec("listshelliter", true, "Iterator name", true);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("listshelliter", true, "Profile : sum", true);
+ exec("deleteshelliter -pn sum -n name", true);
+ exec("listshelliter", true, "Iterator name", false);
+ exec("listshelliter", true, "Iterator xyzzy", true);
+ exec("deleteshelliter -pn sum -a", true);
+ exec("listshelliter", true, "Iterator xyzzy", false);
+ exec("listshelliter", true, "Profile : sum", false);
+ exec("deletetable -f " + table);
+ // list iter
+ exec("createtable " + table);
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ exec("insert a cf cq 1");
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n name", false);
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n other", false);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 11 -n xyzzy", true);
+ exec("scan", true, "3", true);
+ exec("listiter -scan", true, "Iterator name", true);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("listiter -minc", true, "Iterator name", false);
+ exec("listiter -minc", true, "Iterator xyzzy", false);
+ exec("deleteiter -scan -n name", true);
+ exec("listiter -scan", true, "Iterator name", false);
+ exec("listiter -scan", true, "Iterator xyzzy", true);
+ exec("deletetable -f " + table);
+
+ }
+
+ @Test(timeout = 45000)
+ public void setIterOptionPrompt() throws Exception {
+ Connector conn = getConnector();
+ String tableName = name.getMethodName();
+
+ exec("createtable " + tableName);
+ input.set("\n\n");
+ // Setting a non-optiondescriber with no name should fail
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", false);
+
+ // Name as option will work
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+
+ String expectedKey = "table.iterator.scan.cfcounter";
+ String expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ TableOperations tops = conn.tableOperations();
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("customcfcounter\nname1 value1\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30", true);
+ expectedKey = "table.iterator.scan.customcfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name1";
+ expectedValue = "value1";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.customcfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+
+ exec("deletetable " + tableName, true);
+ tableName = tableName + "1";
+
+ exec("createtable " + tableName, true);
+
+ input.set("\nname1 value1.1,value1.2,value1.3\nname2 value2\n\n");
+
+ // Name on the CLI should override OptionDescriber (or user input name, in this case)
+ exec("setiter -scan -class org.apache.accumulo.core.iterators.ColumnFamilyCounter -p 30 -name cfcounter", true);
+ expectedKey = "table.iterator.scan.cfcounter";
+ expectedValue = "30,org.apache.accumulo.core.iterators.ColumnFamilyCounter";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name1";
+ expectedValue = "value1.1,value1.2,value1.3";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ expectedKey = "table.iterator.scan.cfcounter.opt.name2";
+ expectedValue = "value2";
+ checkTableForProperty(tops, tableName, expectedKey, expectedValue);
+ }
+
+ protected void checkTableForProperty(TableOperations tops, String tableName, String expectedKey, String expectedValue) throws Exception {
+ for (int i = 0; i < 5; i++) {
+ for (Entry<String,String> entry : tops.getProperties(tableName)) {
+ if (expectedKey.equals(entry.getKey())) {
+ assertEquals(expectedValue, entry.getValue());
+ return;
+ }
+ }
+ Thread.sleep(500);
+ }
+
+ fail("Failed to find expected property on " + tableName + ": " + expectedKey + "=" + expectedValue);
+ }
+
+ @Test(timeout = 45000)
+ public void notable() throws Exception {
+ final String table = name.getMethodName();
+
+ // notable
+ exec("createtable " + table, true);
+ exec("scan", true, " " + table + ">", true);
+ assertTrue(output.get().contains(" " + table + ">"));
+ exec("notable", true);
+ exec("scan", false, "Not in a table context.", true);
+ assertFalse(output.get().contains(" " + table + ">"));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void sleep() throws Exception {
+ // sleep
+ long now = System.currentTimeMillis();
+ exec("sleep 0.2", true);
+ long diff = System.currentTimeMillis() - now;
+ assertTrue("Diff was actually " + diff, diff >= 200);
+ assertTrue("Diff was actually " + diff, diff < 600);
+ }
+
+ @Test(timeout = 45000)
+ public void addauths() throws Exception {
+ final String table = name.getMethodName();
+ // addauths
+ exec("createtable " + table + " -evc");
- exec("insert a b c d -l foo", false, "does not have authorization", true);
++ exec("insert a b c d -l foo", false, "does not have authorization", true, new ErrorMessageCallback() {
++ public String getErrorMessage() {
++ try {
++ Connector c = getConnector();
++ return "Current auths for root are: " + c.securityOperations().getUserAuthorizations("root").toString();
++ } catch (Exception e) {
++ return "Could not check authorizations";
++ }
++ }
++ });
+ exec("addauths -s foo,bar", true);
+ exec("getauths", true, "foo", true);
+ exec("getauths", true, "bar", true);
+ exec("insert a b c d -l foo");
+ exec("scan", true, "[foo]");
+ exec("scan -s bar", true, "[foo]", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void byeQuitExit() throws Exception {
+ // bye, quit, exit
+ for (String cmd : "bye quit exit".split(" ")) {
+ assertFalse(shell.getExit());
+ exec(cmd);
+ assertTrue(shell.getExit());
+ shell.setExit(false);
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void classpath() throws Exception {
+ // classpath
+ exec("classpath", true, "Level 2: Java Classloader (loads everything defined by java classpath) URL classpath items are", true);
+ }
+
+ @Test(timeout = 45000)
+ public void clearCls() throws Exception {
+ // clear/cls
+ if (shell.getReader().getTerminal().isAnsiSupported()) {
+ exec("cls", true, "[1;1H");
+ exec("clear", true, "[2J");
+ } else {
+ exec("cls", false, "does not support");
+ exec("clear", false, "does not support");
+ }
+ }
+
+ @Test(timeout = 45000)
+ public void clonetable() throws Exception {
+ final String table = name.getMethodName(), clone = table + "_clone";
+
+ // clonetable
+ exec("createtable " + table + " -evc");
+ exec("config -t " + table + " -s table.split.threshold=123M", true);
+ exec("addsplits -t " + table + " a b c", true);
+ exec("insert a b c value");
+ exec("scan", true, "value", true);
+ exec("clonetable " + table + " " + clone);
+ // verify constraint, config, and splits were cloned
+ exec("constraint --list -t " + clone, true, "VisibilityConstraint=2", true);
+ exec("config -t " + clone + " -np", true, "123M", true);
+ exec("getsplits -t " + clone, true, "a\nb\nc\n");
+ exec("deletetable -f " + table);
+ exec("deletetable -f " + clone);
+ }
+
+ @Test(timeout = 45000)
+ public void testCompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ // compact
+ exec("createtable " + table);
+
+ String tableId = getTableId(table);
+
+ // make two files
+ exec("insert a b c d");
+ exec("flush -w");
+ exec("insert x y z v");
+ exec("flush -w");
+ int oldCount = countFiles(tableId);
+ // merge two files into one
+ exec("compact -t " + table + " -w");
+ assertTrue(countFiles(tableId) < oldCount);
+ exec("addsplits -t " + table + " f");
+ // make two more files:
+ exec("insert m 1 2 3");
+ exec("flush -w");
+ exec("insert n 1 2 3");
+ exec("flush -w");
- oldCount = countFiles(tableId);
++ List<String> oldFiles = getFiles(tableId);
+
+ // at this point there are 4 files in the default tablet
- assertEquals(4, oldCount);
++ assertEquals("Files that were found: " + oldFiles, 4, oldFiles.size());
+
+ // compact some data:
+ exec("compact -b g -e z -w");
+ assertEquals(2, countFiles(tableId));
+ exec("compact -w");
+ assertEquals(2, countFiles(tableId));
+ exec("merge --all -t " + table);
+ exec("compact -w");
+ assertEquals(1, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void constraint() throws Exception {
+ final String table = name.getMethodName();
+
+ // constraint
+ exec("constraint -l -t " + MetadataTable.NAME + "", true, "MetadataConstraints=1", true);
+ exec("createtable " + table + " -evc");
+
+ // Make sure the table is fully propagated through zoocache
+ getTableId(table);
+
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", true);
+ exec("constraint -t " + table + " -d 2", true, "Removed constraint 2 from table " + table);
+ // wait for zookeeper updates to propagate
+ UtilWaitThread.sleep(1000);
+ exec("constraint -l -t " + table, true, "VisibilityConstraint=2", false);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deletemany() throws Exception {
+ final String table = name.getMethodName();
+
+ // deletemany
+ exec("createtable " + table);
+ make10();
+ assertEquals(10, countkeys(table));
+ exec("deletemany -f -b row8");
+ assertEquals(8, countkeys(table));
+ exec("scan -t " + table + " -np", true, "row8", false);
+ make10();
+ exec("deletemany -f -b row4 -e row5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -c cf:col4,cf:col5");
+ assertEquals(8, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -r row3");
+ assertEquals(9, countkeys(table));
+ make10();
+ exec("deletemany -f -b row3 -be -e row5 -ee");
+ assertEquals(9, countkeys(table));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void deleterows() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ final String tableId = getTableId(table);
+
+ // deleterows
+ int base = countFiles(tableId);
+ assertEquals(0, base);
+
+ exec("addsplits row5 row7");
+ make10();
+ exec("flush -w -t " + table);
- assertEquals(3, countFiles(tableId));
++ List<String> files = getFiles(tableId);
++ assertEquals("Found the following files: " + files, 3, files.size());
+ exec("deleterows -t " + table + " -b row5 -e row7", true);
+ assertEquals(2, countFiles(tableId));
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void groups() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("setgroups -t " + table + " alpha=a,b,c num=3,2,1");
+ exec("getgroups -t " + table, true, "alpha=a,b,c", true);
+ exec("getgroups -t " + table, true, "num=1,2,3", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void grep() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ make10();
+ exec("grep row[123]", true, "row1", false);
+ exec("grep row5", true, "row5", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void help() throws Exception {
+ exec("help -np", true, "Help Commands", true);
+ exec("?", true, "Help Commands", true);
+ for (String c : ("bye exit quit " + "about help info ? " + "deleteiter deletescaniter listiter setiter setscaniter "
+ + "grant revoke systempermissions tablepermissions userpermissions " + "execfile history " + "authenticate cls clear notable sleep table user whoami "
+ + "clonetable config createtable deletetable droptable du exporttable importtable offline online renametable tables "
+ + "addsplits compact constraint flush getgropus getsplits merge setgroups " + "addauths createuser deleteuser dropuser getauths passwd setauths users "
+ + "delete deletemany deleterows egrep formatter interpreter grep importdirectory insert maxrow scan").split(" ")) {
+ exec("help " + c, true);
+ }
+ }
+
+ // @Test(timeout = 45000)
+ public void history() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("history -c", true);
+ exec("createtable " + table);
+ exec("deletetable -f " + table);
+ exec("history", true, table, true);
+ exec("history", true, "history", true);
+ }
+
+ @Test(timeout = 45000)
+ public void importDirectory() throws Exception {
+ final String table = name.getMethodName();
+
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ File importDir = new File(getFolder(), "import");
+ importDir.mkdir();
+ String even = new File(importDir, "even.rf").toString();
+ String odd = new File(importDir, "odd.rf").toString();
+ File errorsDir = new File(getFolder(), "errors");
+ errorsDir.mkdir();
+ fs.mkdirs(new Path(errorsDir.toString()));
+ AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration();
+ FileSKVWriter evenWriter = FileOperations.getInstance().openWriter(even, fs, conf, aconf);
+ evenWriter.startDefaultLocalityGroup();
+ FileSKVWriter oddWriter = FileOperations.getInstance().openWriter(odd, fs, conf, aconf);
+ oddWriter.startDefaultLocalityGroup();
+ long ts = System.currentTimeMillis();
+ Text cf = new Text("cf");
+ Text cq = new Text("cq");
+ Value value = new Value("value".getBytes());
+ for (int i = 0; i < 100; i += 2) {
+ Key key = new Key(new Text(String.format("%8d", i)), cf, cq, ts);
+ evenWriter.append(key, value);
+ key = new Key(new Text(String.format("%8d", i + 1)), cf, cq, ts);
+ oddWriter.append(key, value);
+ }
+ evenWriter.close();
+ oddWriter.close();
+ assertEquals(0, shell.getExitCode());
+ exec("createtable " + table, true);
+ exec("importdirectory " + importDir + " " + errorsDir + " true", true);
+ exec("scan -r 00000000", true, "00000000", true);
+ exec("scan -r 00000099", true, "00000099", true);
+ exec("deletetable -f " + table);
+ }
+
+ @Test(timeout = 45000)
+ public void info() throws Exception {
+ exec("info", true, Constants.VERSION, true);
+ }
+
+ @Test(timeout = 45000)
+ public void interpreter() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("interpreter -l", true, "HexScan", false);
+ exec("insert \\x02 cf cq value", true);
+ exec("scan -b 02", true, "value", false);
+ exec("interpreter -i org.apache.accumulo.core.util.interpret.HexScanInterpreter", true);
+ // Need to allow time for this to propagate through zoocache/zookeeper
+ UtilWaitThread.sleep(3000);
+
+ exec("interpreter -l", true, "HexScan", true);
+ exec("scan -b 02", true, "value", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void listcompactions() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s table.iterator.minc.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
+ exec("config -t " + table + " -s table.iterator.minc.slow.opt.sleepTime=1000", true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert c cf cq value", true);
+ exec("insert d cf cq value", true);
+ exec("flush -t " + table, true);
+ exec("sleep 0.2", true);
+ exec("listcompactions", true, "default_tablet");
+ String[] lines = output.get().split("\n");
+ String last = lines[lines.length - 1];
+ String[] parts = last.split("\\|");
+ assertEquals(12, parts.length);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void maxrow() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
+ exec("insert a cf cq value", true);
+ exec("insert b cf cq value", true);
+ exec("insert ccc cf cq value", true);
+ exec("insert zzz cf cq value", true);
+ exec("maxrow", true, "zzz", true);
+ exec("delete zzz cf cq", true);
+ exec("maxrow", true, "ccc", true);
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void merge() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table);
+ exec("addsplits a m z");
+ exec("getsplits", true, "z", true);
+ exec("merge --all", true);
+ exec("getsplits", true, "z", false);
+ exec("deletetable -f " + table);
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(2, output.get().split("\n").length);
+ exec("getsplits -t accumulo.root", true);
+ assertEquals(1, output.get().split("\n").length);
+ exec("merge --all -t " + MetadataTable.NAME + "");
+ exec("getsplits -t " + MetadataTable.NAME + "", true);
+ assertEquals(1, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void ping() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ exec("ping", true, "OK", true);
+ // wait for both tservers to start up
+ if (output.get().split("\n").length == 3)
+ break;
+ UtilWaitThread.sleep(1000);
+
+ }
+ assertEquals(3, output.get().split("\n").length);
+ }
+
+ @Test(timeout = 45000)
+ public void renametable() throws Exception {
+ final String table = name.getMethodName() + "1", rename = name.getMethodName() + "2";
+
+ exec("createtable " + table);
+ exec("insert this is a value");
+ exec("renametable " + table + " " + rename);
+ exec("tables", true, rename, true);
+ exec("tables", true, table, false);
+ exec("scan -t " + rename, true, "value", true);
+ exec("deletetable -f " + rename, true);
+ }
+
+ @Test(timeout = 30000)
+ public void tables() throws Exception {
+ final String table = name.getMethodName(), table1 = table + "_z", table2 = table + "_a";
+ exec("createtable " + table1);
+ exec("createtable " + table2);
+ exec("notable");
+ String lst = exec("tables -l");
+ assertTrue(lst.indexOf(table2) < lst.indexOf(table1));
+ lst = exec("tables -l -s");
+ assertTrue(lst.indexOf(table1) < lst.indexOf(table2));
+ }
+
+ @Test(timeout = 45000)
+ public void systempermission() throws Exception {
+ exec("systempermissions");
+ assertEquals(11, output.get().split("\n").length - 1);
+ exec("tablepermissions", true);
+ assertEquals(6, output.get().split("\n").length - 1);
+ }
+
+ @Test(timeout = 45000)
+ public void listscans() throws Exception {
+ final String table = name.getMethodName();
+
+ exec("createtable " + table, true);
++
++ // Should be about a 3 second scan
++ for (int i = 0; i < 6; i++) {
++ exec("insert " + i + " cf cq value", true);
++ }
+ exec("config -t " + table + " -s table.iterator.scan.slow=30,org.apache.accumulo.test.functional.SlowIterator", true);
- exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=1000", true);
- exec("insert a cf cq value", true);
- exec("insert b cf cq value", true);
- exec("insert c cf cq value", true);
- exec("insert d cf cq value", true);
++ exec("config -t " + table + " -s table.iterator.scan.slow.opt.sleepTime=500", true);
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Connector connector = getConnector();
+ Scanner s = connector.createScanner(table, Authorizations.EMPTY);
+ FunctionalTestUtils.count(s);
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+ thread.start();
- exec("sleep 0.1", true);
- String scans = exec("listscans", true);
- String lines[] = scans.split("\n");
- String last = lines[lines.length - 1];
- assertTrue(last.contains("RUNNING"));
- String parts[] = last.split("\\|");
- assertEquals(13, parts.length);
- String hostPortPattern = ".+:\\d+";
- String tserver = parts[0].trim();
- assertTrue(tserver.matches(hostPortPattern));
- assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
- String client = parts[1].trim();
- assertTrue(client.matches(hostPortPattern));
- // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
++
++ List<String> scans = new ArrayList<String>();
++ // Try to find the active scan for about 5seconds
++ for (int i = 0; i < 50 && scans.isEmpty(); i++) {
++ String currentScans = exec("listscans", true);
++ String[] lines = currentScans.split("\n");
++ for (int scanOffset = 2; i < lines.length; i++) {
++ String currentScan = lines[scanOffset];
++ if (currentScan.contains(table)) {
++ scans.add(currentScan);
++ }
++ }
++ UtilWaitThread.sleep(100);
++ }
+ thread.join();
++
++ assertFalse("Could not find any active scans over table " + table, scans.isEmpty());
++
++ for (String scan : scans) {
++ assertTrue("Scan does not appear to be a 'RUNNING' scan: '" + scan + "'", scan.contains("RUNNING"));
++ String parts[] = scan.split("\\|");
++ assertEquals("Expected 13 colums, but found " + parts.length + " instead for '" + Arrays.toString(parts) + "'", 13, parts.length);
++ String tserver = parts[0].trim();
++ // TODO: any way to tell if the client address is accurate? could be local IP, host, loopback...?
++ String hostPortPattern = ".+:\\d+";
++ assertTrue(tserver.matches(hostPortPattern));
++ assertTrue(getConnector().instanceOperations().getTabletServers().contains(tserver));
++ String client = parts[1].trim();
++ assertTrue(client.matches(hostPortPattern));
++ }
++
+ exec("deletetable -f " + table, true);
+ }
+
+ @Test(timeout = 45000)
+ public void testPertableClasspath() throws Exception {
+ final String table = name.getMethodName();
+
+ File fooFilterJar = File.createTempFile("FooFilter", ".jar", getFolder());
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooFilter.jar"), fooFilterJar);
+ fooFilterJar.deleteOnExit();
+
+ File fooConstraintJar = File.createTempFile("FooConstraint", ".jar", getFolder());
+ FileUtils.copyURLToFile(this.getClass().getResource("/FooConstraint.jar"), fooConstraintJar);
+ fooConstraintJar.deleteOnExit();
+
+ exec(
+ "config -s " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1=" + fooFilterJar.toURI().toString() + "," + fooConstraintJar.toURI().toString(),
+ true);
+
+ exec("createtable " + table, true);
+ exec("config -t " + table + " -s " + Property.TABLE_CLASSPATH.getKey() + "=cx1", true);
+
+ UtilWaitThread.sleep(200);
+
+ // We can't use the setiter command as Filter implements OptionDescriber which
+ // forces us to enter more input that I don't know how to input
+ // Instead, we can just manually set the property on the table.
+ exec("config -t " + table + " -s " + Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.foo=10,org.apache.accumulo.test.FooFilter");
+
+ exec("insert foo f q v", true);
+
+ UtilWaitThread.sleep(100);
+
+ exec("scan -np", true, "foo", false);
+
+ exec("constraint -a FooConstraint", true);
+
+ exec("offline -w " + table);
+ exec("online -w " + table);
+
+ exec("table " + table, true);
+ exec("insert foo f q v", false);
+ exec("insert ok foo q v", true);
+
+ exec("deletetable -f " + table, true);
+ exec("config -d " + Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1");
+
+ }
+
+ @Test(timeout = 45000)
+ public void trace() throws Exception {
+ // Make sure to not collide with the "trace" table
+ final String table = name.getMethodName() + "Test";
+
+ exec("trace on", true);
+ exec("createtable " + table, true);
+ exec("insert a b c value", true);
+ exec("scan -np", true, "value", true);
+ exec("deletetable -f " + table);
+ exec("sleep 1");
+ String trace = exec("trace off");
+ System.out.println(trace);
+ assertTrue(trace.contains("sendMutations"));
+ assertTrue(trace.contains("startScan"));
+ assertTrue(trace.contains("DeleteTable"));
+ }
+
+ @Test(timeout = 30000)
+ public void badLogin() throws Exception {
+ input.set(ROOT_PASSWORD + "\n");
+ String err = exec("user NoSuchUser", false);
+ assertTrue(err.contains("BAD_CREDENTIALS for user NoSuchUser"));
+ }
+
+ @Test(timeout = 30000)
+ public void namespaces() throws Exception {
+ exec("namespaces", true, "\"\"", true); // default namespace, displayed as quoted empty string
+ exec("namespaces", true, Namespaces.ACCUMULO_NAMESPACE, true);
+ exec("createnamespace thing1", true);
+ String namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing1"));
+
+ exec("renamenamespace thing1 thing2");
+ namespaces = exec("namespaces");
+ assertTrue(namespaces.contains("thing2"));
+ assertTrue(!namespaces.contains("thing1"));
+
+ // can't delete a namespace that still contains tables, unless you do -f
+ exec("createtable thing2.thingy", true);
+ exec("deletenamespace thing2");
+ exec("y");
+ exec("namespaces", true, "thing2", true);
+
+ exec("du -ns thing2", true, "thing2.thingy", true);
+
+ // all "TableOperation" commands can take a namespace
+ exec("offline -ns thing2", true);
+ exec("online -ns thing2", true);
+ exec("flush -ns thing2", true);
+ exec("compact -ns thing2", true);
+ exec("createnamespace testers3", true);
+ exec("createtable testers3.1", true);
+ exec("createtable testers3.2", true);
+ exec("deletetable -ns testers3 -f", true);
+ exec("tables", true, "testers3.1", false);
+ exec("namespaces", true, "testers3", true);
+ exec("deletenamespace testers3 -f", true);
+ input.set("true\n\n\nSTRING\n");
+ exec("setiter -ns thing2 -scan -class org.apache.accumulo.core.iterators.user.SummingCombiner -p 10 -n name", true);
+ exec("listiter -ns thing2 -scan", true, "Summing", true);
+ exec("deleteiter -ns thing2 -n name -scan", true);
+ exec("createuser dude");
+ exec("pass");
+ exec("pass");
+ exec("grant Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+ exec("revoke Namespace.CREATE_TABLE -ns thing2 -u dude", true);
+
+ // properties override and such
+ exec("config -ns thing2 -s table.file.max=44444", true);
+ exec("config -ns thing2", true, "44444", true);
+ exec("config -t thing2.thingy", true, "44444", true);
+ exec("config -t thing2.thingy -s table.file.max=55555", true);
+ exec("config -t thing2.thingy", true, "55555", true);
+
+ // can copy properties when creating
+ exec("createnamespace thing3 -cc thing2", true);
+ exec("config -ns thing3", true, "44444", true);
+
+ exec("deletenamespace -f thing2", true);
+ exec("namespaces", true, "thing2", false);
+ exec("tables", true, "thing2.thingy", false);
+
+ // put constraints on a namespace
+ exec("constraint -ns thing3 -a org.apache.accumulo.examples.simple.constraints.NumericValueConstraint", true);
+ exec("createtable thing3.constrained", true);
+ exec("table thing3.constrained", true);
+ exec("constraint -d 1");
+ // should fail
+ exec("constraint -l", true, "NumericValueConstraint", true);
+ exec("insert r cf cq abc", false);
+ exec("constraint -ns thing3 -d 1");
+ exec("sleep 1");
+ exec("insert r cf cq abc", true);
+ }
+
+ private int countkeys(String table) throws IOException {
+ exec("scan -np -t " + table);
+ return output.get().split("\n").length - 1;
+ }
+
+ @Test(timeout = 30000)
+ public void scans() throws Exception {
+ exec("createtable t");
+ make10();
+ String result = exec("scan -np -b row1 -e row1");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row3 -e row5");
+ assertEquals(4, result.split("\n").length);
+ result = exec("scan -np -r row3");
+ assertEquals(2, result.split("\n").length);
+ result = exec("scan -np -b row:");
+ assertEquals(1, result.split("\n").length);
+ result = exec("scan -np -b row");
+ assertEquals(11, result.split("\n").length);
+ result = exec("scan -np -e row:");
+ assertEquals(11, result.split("\n").length);
+ exec("deletetable -f t");
+ }
+
+ @Test(timeout = 30000)
+ public void whoami() throws Exception {
+ assertTrue(exec("whoami", true).contains("root"));
+ input.set("secret\nsecret\n");
+ exec("createuser test_user");
+ exec("setauths -u test_user -s 12,3,4");
+ String auths = exec("getauths -u test_user");
+ assertTrue(auths.contains("3") && auths.contains("12") && auths.contains("4"));
+ input.set("secret\n");
+ exec("user test_user", true);
+ assertTrue(exec("whoami", true).contains("test_user"));
+ input.set(ROOT_PASSWORD + "\n");
+ exec("user root", true);
+ }
+
+ private void make10() throws IOException {
+ for (int i = 0; i < 10; i++) {
+ exec(String.format("insert row%d cf col%d value", i, i));
+ }
+ }
++
++ private List<String> getFiles(String tableId) throws IOException {
++ output.clear();
+
- private int countFiles(String tableId) throws IOException {
+ exec("scan -t " + MetadataTable.NAME + " -np -c file -b " + tableId + " -e " + tableId + "~");
+
+ log.debug("countFiles(): " + output.get());
+
- return output.get().split("\n").length - 1;
++ String[] lines = StringUtils.split(output.get(), "\n");
++ output.clear();
++
++ if (0 == lines.length) {
++ return Collections.emptyList();
++ }
++
++ return Arrays.asList(Arrays.copyOfRange(lines, 1, lines.length));
++ }
++
++ private int countFiles(String tableId) throws IOException {
++ return getFiles(tableId).size();
+ }
+
+ private String getTableId(String tableName) throws Exception {
+ Connector conn = getConnector();
+
+ for (int i = 0; i < 5; i++) {
+ Map<String,String> nameToId = conn.tableOperations().tableIdMap();
+ if (nameToId.containsKey(tableName)) {
+ return nameToId.get(tableName);
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+
+ fail("Could not find ID for table: " + tableName);
+ // Will never get here
+ return null;
+ }
+
+}
[20/23] git commit: ACCUMULO-2061 Don't store the defaultVolume in
with the rest but expect it in the constructor.
Posted by el...@apache.org.
ACCUMULO-2061 Don't store the defaultVolume in with the rest but expect it in the constructor.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c3214e0e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c3214e0e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c3214e0e
Branch: refs/heads/ACCUMULO-2061
Commit: c3214e0e9433fc35cc822671ba66286e15d38f22
Parents: 3eb1593
Author: Josh Elser <el...@apache.org>
Authored: Tue Mar 11 01:00:50 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Tue Mar 11 01:00:50 2014 -0400
----------------------------------------------------------------------
.../core/volume/VolumeConfiguration.java | 3 +++
.../accumulo/server/client/HdfsZooInstance.java | 1 +
.../accumulo/server/fs/VolumeManager.java | 12 ++++++++++--
.../accumulo/server/fs/VolumeManagerImpl.java | 20 ++++++++++++--------
.../tserver/TabletServerSyncCheckTest.java | 2 +-
.../java/org/apache/accumulo/test/VolumeIT.java | 20 +++++++++++---------
6 files changed, 38 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c3214e0e/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
index 3f4ab3e..3005174 100644
--- a/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/volume/VolumeConfiguration.java
@@ -46,6 +46,9 @@ public class VolumeConfiguration {
public static Volume getDefaultVolume(Configuration conf, AccumuloConfiguration acuconf) throws IOException {
@SuppressWarnings("deprecation")
String uri = acuconf.get(Property.INSTANCE_DFS_URI);
+
+ // By default pull from INSTANCE_DFS_URI, falling back to the Hadoop defined
+ // default filesystem (fs.defaultFS or the deprecated fs.default.name)
if ("".equals(uri))
return create(FileSystem.get(conf), acuconf);
else
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c3214e0e/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index 8aa0be9..6bff9c5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@ -135,6 +135,7 @@ public class HdfsZooInstance implements Instance {
throw new RuntimeException(e);
}
Volume randVolume = fs.getVolumes().iterator().next();
+ log.info("Looking for instanceId from " + randVolume);
String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(ServerConstants.getInstanceIdLocation(randVolume), acuConf);
instanceId = instanceIdFromFile;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c3214e0e/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
index 2f7cf42..c345ede 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
@@ -156,9 +156,17 @@ public interface VolumeManager {
// decide on which of the given locations to create a new file
String choose(String[] options);
-
+
+ /**
+ * Fetch the default Volume
+ * @return
+ */
public Volume getDefaultVolume();
-
+
+ /**
+ * Fetch the configured Volumes, excluding the default Volume
+ * @return
+ */
public Collection<Volume> getVolumes();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c3214e0e/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
index a32b0bd..ca5167d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
@@ -70,9 +70,9 @@ public class VolumeManagerImpl implements VolumeManager {
AccumuloConfiguration conf;
VolumeChooser chooser;
- protected VolumeManagerImpl(Map<String,Volume> volumes, String defaultVolume, AccumuloConfiguration conf) {
+ protected VolumeManagerImpl(Map<String,Volume> volumes, Volume defaultVolume, AccumuloConfiguration conf) {
this.volumesByName = volumes;
- this.defaultVolume = volumes.get(defaultVolume);
+ this.defaultVolume = defaultVolume;
// We may have multiple directories used in a single FileSystem (e.g. testing)
this.volumesByFileSystem = HashMultimap.create();
invertVolumesByFileSystem(volumesByName, volumesByFileSystem);
@@ -91,7 +91,7 @@ public class VolumeManagerImpl implements VolumeManager {
AccumuloConfiguration accConf = DefaultConfiguration.getDefaultConfiguration();
Volume defaultLocalVolume = VolumeConfiguration.create(FileSystem.getLocal(CachedConfiguration.getInstance()), localBasePath);
- return new VolumeManagerImpl(Collections.singletonMap("", defaultLocalVolume), "", accConf);
+ return new VolumeManagerImpl(Collections.singletonMap(DEFAULT, defaultLocalVolume), defaultLocalVolume, accConf);
}
@Override
@@ -306,11 +306,16 @@ public class VolumeManagerImpl implements VolumeManager {
return candidateVolume;
}
}
-
- throw new RuntimeException("Could not determine valid Volume for Path '" + path + "' from Volumes " + candidateVolumes);
+
+ // For the same reason as we can have multiple Volumes within a single filesystem
+ // we could also not find a matching one. We should defer back to the defaultVolume
+ // e.g. volume rename with old path references
+ log.debug("Defaulting to " + defaultVolume + " as a valid volume could not be determined for " + path);
+
+ return defaultVolume;
}
- log.info("Could not determine Volume for Path: " + path);
+ log.debug("Could not determine volume for Path '" + path + "' from defined volumes");
} catch (IOException ex) {
throw new RuntimeException(ex);
}
@@ -397,7 +402,6 @@ public class VolumeManagerImpl implements VolumeManager {
final Configuration hadoopConf = CachedConfiguration.getInstance();
// The "default" Volume for Accumulo (in case no volumes are specified)
- volumes.put(DEFAULT, VolumeConfiguration.getDefaultVolume(hadoopConf, conf));
for (String volumeUriOrDir : VolumeConfiguration.getVolumeUris(conf)) {
if (volumeUriOrDir.equals(DEFAULT))
// Cannot re-define the default volume
@@ -411,7 +415,7 @@ public class VolumeManagerImpl implements VolumeManager {
}
}
- return new VolumeManagerImpl(volumes, DEFAULT, conf);
+ return new VolumeManagerImpl(volumes, VolumeConfiguration.getDefaultVolume(hadoopConf, conf), conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c3214e0e/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
index a5a1deb..dad9a75 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java
@@ -92,7 +92,7 @@ public class TabletServerSyncCheckTest {
public TestVolumeManagerImpl(Map<String,Volume> volumes) {
- super(volumes, volumes.keySet().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
+ super(volumes, volumes.values().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap()));
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c3214e0e/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
index a0efe45..7ac34f8 100644
--- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java
@@ -23,6 +23,7 @@ import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -104,8 +105,9 @@ public class VolumeIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
// Run MAC on two locations in the local file system
- cfg.setProperty(Property.INSTANCE_DFS_URI, v1.toString());
- cfg.setProperty(Property.INSTANCE_DFS_DIR, "/accumulo");
+ URI v1Uri = v1.toUri();
+ cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
+ cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
// use raw local file system so walogs sync and flush will work
@@ -114,7 +116,7 @@ public class VolumeIT extends ConfigurableMacIT {
super.configure(cfg, hadoopCoreSite);
}
- @Test
+ @Test(timeout = 2 * 60 * 1000)
public void test() throws Exception {
// create a table
Connector connector = getConnector();
@@ -176,7 +178,7 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(expected, actual);
}
- @Test
+ @Test(timeout = 2 * 60 * 1000)
public void testRelativePaths() throws Exception {
List<String> expected = new ArrayList<String>();
@@ -390,7 +392,7 @@ public class VolumeIT extends ConfigurableMacIT {
Assert.assertEquals(200, sum);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testRemoveVolumes() throws Exception {
String[] tableNames = getTableNames(2);
@@ -447,12 +449,12 @@ public class VolumeIT extends ConfigurableMacIT {
File v1f = new File(v1.toUri());
File v8f = new File(new File(v1.getParent().toUri()), "v8");
- v1f.renameTo(v8f);
+ Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
Path v8 = new Path(v8f.toURI());
File v2f = new File(v2.toUri());
File v9f = new File(new File(v2.getParent().toUri()), "v9");
- v2f.renameTo(v9f);
+ Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
Path v9 = new Path(v9f.toURI());
Configuration conf = new Configuration(false);
@@ -494,12 +496,12 @@ public class VolumeIT extends ConfigurableMacIT {
verifyVolumesUsed(tableNames[2], true, v8, v9);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testCleanReplaceVolumes() throws Exception {
testReplaceVolume(true);
}
- @Test
+ @Test(timeout = 5 * 60 * 1000)
public void testDirtyReplaceVolumes() throws Exception {
testReplaceVolume(false);
}
[09/23] git commit: ACCUMULO-2005: Add a 3rd argument to functional
test RunTests.java to allow a timeout factor for mapred.task.timeout and
run.py -f flag
Posted by el...@apache.org.
ACCUMULO-2005: Add a 3rd argument to functional test RunTests.java to allow a timeout factor for mapred.task.timeout and run.py -f flag
Signed-off-by: Sean Busbey <bu...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6faac421
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6faac421
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6faac421
Branch: refs/heads/ACCUMULO-2061
Commit: 6faac421b560c9ce92f1d4db3c0b60d830498a71
Parents: 4fd48fd
Author: Hung Pham <hp...@cloudera.com>
Authored: Fri Jan 17 09:36:44 2014 -0500
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Mon Mar 10 09:05:46 2014 -0500
----------------------------------------------------------------------
.../server/test/functional/RunTests.java | 27 +++++++++++++++++---
test/system/auto/README | 10 ++++++++
2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6faac421/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java b/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
index 4dac215..31e3198 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/functional/RunTests.java
@@ -73,12 +73,15 @@ public class RunTests extends Configured implements Tool {
private static final Logger log = Logger.getLogger(RunTests.class);
private Job job = null;
-
+ static private Integer timeoutFactor = 1;
+ static final String TIMEOUT_FACTOR = RunTests.class.getName() + ".timeoutFactor";
+
static public class TestMapper extends Mapper<LongWritable,Text,Text,Text> {
private static final String REDUCER_RESULT_START = "::::: ";
private static final int RRS_LEN = REDUCER_RESULT_START.length();
private Text result = new Text();
+ String mapperTimeoutFactor = null;
private static enum Outcome {
SUCCESS, FAILURE, ERROR, UNEXPECTED_SUCCESS, EXPECTED_FAILURE
@@ -95,7 +98,7 @@ public class RunTests extends Configured implements Tool {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
- List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-t", value.toString());
+ List<String> cmd = Arrays.asList("/usr/bin/python", "test/system/auto/run.py", "-m", "-f", mapperTimeoutFactor, "-t", value.toString());
log.info("Running test " + cmd);
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.directory(new File(context.getConfiguration().get("accumulo.home")));
@@ -129,7 +132,11 @@ public class RunTests extends Configured implements Tool {
p.waitFor();
}
-
+
+ @Override
+ protected void setup(Mapper.Context context) throws IOException, InterruptedException {
+ mapperTimeoutFactor = Integer.toString(context.getConfiguration().getInt(TIMEOUT_FACTOR, timeoutFactor));
+ }
}
@Override
@@ -141,7 +148,11 @@ public class RunTests extends Configured implements Tool {
Configuration conf = job.getConfiguration();
conf.setInt("mapred.max.split.size", 40);
conf.set("accumulo.home", System.getenv("ACCUMULO_HOME"));
- conf.setInt("mapred.task.timeout", 8 * 60 * 1000);
+
+ // Taking third argument as scaling factor to setting mapred.task.timeout
+ // and TIMEOUT_FACTOR
+ conf.setInt("mapred.task.timeout", timeoutFactor * 8 * 60 * 1000);
+ conf.setInt(TIMEOUT_FACTOR, timeoutFactor);
conf.setBoolean("mapred.map.tasks.speculative.execution", false);
// set input
@@ -179,6 +190,14 @@ public class RunTests extends Configured implements Tool {
* @throws Exception
*/
public static void main(String[] args) throws Exception {
+ if (args.length > 2) {
+ try {
+ timeoutFactor = Integer.parseInt(args[2]);
+ } catch (NumberFormatException e) {
+ log.error("timeoutFactor must be an integer: ", e);
+ System.exit(1);
+ }
+ }
RunTests tests = new RunTests();
ToolRunner.run(new Configuration(), tests, args);
tests.job.waitForCompletion(true);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6faac421/test/system/auto/README
----------------------------------------------------------------------
diff --git a/test/system/auto/README b/test/system/auto/README
index 45ed158..7722c63 100644
--- a/test/system/auto/README
+++ b/test/system/auto/README
@@ -69,6 +69,16 @@ cluster at your disposal, you can run the tests as a MapReduce job:
The example above runs every test. You can trim the tests file to include
only the tests you wish to run.
+You may specify a 'timeout factor' via an optional integer as a third argument:
+
+ $ ./bin/accumulo org.apache.accumulo.server.test.functional.RunTests \
+ /user/hadoop/tests /user/hadoop/results timeout_factor
+
+Where 'timeout_factor' indicates how much we should scale up timeouts. It will
+be used to set both mapred.task.timeout and the "-f" flag used by run.py. If
+not given, 'timeout_factor' defaults to 1, which corresponds to a
+mapred.task.timeout of 480 seconds.
+
In some clusters, the user under which MR jobs run is different from the user
under which Accumulo is installed, and this can cause failures running the
tests. Various configuration and permission changes can be made to help the