You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/12/12 01:27:09 UTC
[4/4] git commit: ACCUMULO-1599 Stop using /tmp wherever possible
ACCUMULO-1599 Stop using /tmp wherever possible
The only obvious remaining place I found was the Jetty temporary
directories created by Hadoop MiniDFSCluster. These get cleaned up
automatically when the tests pass and don't really offer much value
for debugging test failures, so I don't consider it a high priority.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c1fbeac5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c1fbeac5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c1fbeac5
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: c1fbeac505ad1d01da656f597487173dfb511ded
Parents: 9695621
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Dec 11 18:20:01 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Dec 11 19:24:17 2013 -0500
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 11 +-
.../apache/accumulo/core/data/KeyExtent.java | 276 ++++-----
.../accumulo/core/util/ArgumentChecker.java | 18 +-
.../client/mapred/AccumuloInputFormatTest.java | 31 +-
.../core/client/mock/MockNamespacesTest.java | 3 +-
.../simple/client/RandomBatchWriter.java | 48 +-
.../simple/filedata/ChunkInputFormatTest.java | 28 +-
.../minicluster/MiniAccumuloCluster.java | 26 +-
.../security/AuditedSecurityOperation.java | 120 ++--
.../tserver/log/TestUpgradePathForWALogs.java | 44 +-
.../accumulo/test/MultiTableBatchWriterIT.java | 505 +++++++++++++++++
.../test/MultiTableBatchWriterTest.java | 565 -------------------
.../org/apache/accumulo/test/ShellServerIT.java | 5 +-
.../test/functional/ConfigurableMacIT.java | 2 +-
.../accumulo/test/functional/ExamplesIT.java | 112 ++--
.../test/functional/FunctionalTestUtils.java | 45 +-
.../accumulo/test/functional/MapReduceIT.java | 7 +-
.../accumulo/test/functional/SimpleMacIT.java | 3 +
.../accumulo/test/functional/ZooCacheIT.java | 16 +-
.../accumulo/test/util/CertUtilsTest.java | 2 +-
20 files changed, 916 insertions(+), 951 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 47f9c37..2ab3a20 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -84,16 +84,15 @@ public enum Property {
@Sensitive
RPC_SSL_KEYSTORE_PASSWORD("rpc.javax.net.ssl.keyStorePassword", "", PropertyType.STRING,
"Password used to encrypt the SSL private keystore. Leave blank to use the Accumulo instance secret"),
- RPC_SSL_KEYSTORE_TYPE("rpc.javax.net.ssl.keyStoreType", "jks", PropertyType.STRING,
- "Type of SSL keystore"),
+ RPC_SSL_KEYSTORE_TYPE("rpc.javax.net.ssl.keyStoreType", "jks", PropertyType.STRING, "Type of SSL keystore"),
RPC_SSL_TRUSTSTORE_PATH("rpc.javax.net.ssl.trustStore", "$ACCUMULO_CONF_DIR/ssl/truststore.jks", PropertyType.PATH,
"Path of the truststore file for the root cert"),
@Sensitive
RPC_SSL_TRUSTSTORE_PASSWORD("rpc.javax.net.ssl.trustStorePassword", "", PropertyType.STRING,
"Password used to encrypt the SSL truststore. Leave blank to use no password"),
- RPC_SSL_TRUSTSTORE_TYPE("rpc.javax.net.ssl.trustStoreType", "jks", PropertyType.STRING,
- "Type of SSL truststore"),
- RPC_USE_JSSE("rpc.useJsse", "false", PropertyType.BOOLEAN, "Use JSSE system properties to configure SSL rather than general.javax.net.ssl.* Accumulo properties"),
+ RPC_SSL_TRUSTSTORE_TYPE("rpc.javax.net.ssl.trustStoreType", "jks", PropertyType.STRING, "Type of SSL truststore"),
+ RPC_USE_JSSE("rpc.useJsse", "false", PropertyType.BOOLEAN,
+ "Use JSSE system properties to configure SSL rather than general.javax.net.ssl.* Accumulo properties"),
// instance properties (must be the same for every node in an instance)
INSTANCE_PREFIX("instance.", null, PropertyType.PREFIX,
"Properties in this category must be consistent throughout a cloud. This is enforced and servers won't be able to communicate if these differ."),
@@ -598,7 +597,7 @@ public enum Property {
}
// This is not a cache for loaded classes, just a way to avoid spamming the debug log
- static Map<String, Class<? extends Object>> loaded = Collections.synchronizedMap(new HashMap<String, Class<? extends Object>>());
+ static Map<String,Class<?>> loaded = Collections.synchronizedMap(new HashMap<String,Class<?>>());
public static <T> T createInstanceFromPropertyName(AccumuloConfiguration conf, Property property, Class<T> base, T defaultInstance) {
String clazzName = conf.get(property);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
index fc766fe..9dc8db5 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
@@ -51,9 +51,9 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
public class KeyExtent implements WritableComparable<KeyExtent> {
-
+
private static final WeakHashMap<Text,WeakReference<Text>> tableIds = new WeakHashMap<Text,WeakReference<Text>>();
-
+
private static Text dedupeTableId(Text tableId) {
synchronized (tableIds) {
WeakReference<Text> etir = tableIds.get(tableId);
@@ -63,30 +63,30 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
return eti;
}
}
-
+
tableId = new Text(tableId);
tableIds.put(tableId, new WeakReference<Text>(tableId));
return tableId;
}
}
-
+
private Text textTableId;
private Text textEndRow;
private Text textPrevEndRow;
-
+
private void check() {
-
+
if (getTableId() == null)
throw new IllegalArgumentException("null table id not allowed");
-
+
if (getEndRow() == null || getPrevEndRow() == null)
return;
-
+
if (getPrevEndRow().compareTo(getEndRow()) >= 0) {
throw new IllegalArgumentException("prevEndRow (" + getPrevEndRow() + ") >= endRow (" + getEndRow() + ")");
}
}
-
+
/**
* Default constructor
*
@@ -96,32 +96,32 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
this.setEndRow(new Text(), false, false);
this.setPrevEndRow(new Text(), false, false);
}
-
+
public KeyExtent(Text table, Text endRow, Text prevEndRow) {
this.setTableId(table);
this.setEndRow(endRow, false, true);
this.setPrevEndRow(prevEndRow, false, true);
-
+
check();
}
-
+
public KeyExtent(KeyExtent extent) {
// extent has already deduped table id, so there is no need to do it again
this.textTableId = extent.textTableId;
this.setEndRow(extent.getEndRow(), false, true);
this.setPrevEndRow(extent.getPrevEndRow(), false, true);
-
+
check();
}
-
+
public KeyExtent(TKeyExtent tke) {
this.setTableId(new Text(ByteBufferUtil.toBytes(tke.table)));
this.setEndRow(tke.endRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.endRow)), false, false);
this.setPrevEndRow(tke.prevEndRow == null ? null : new Text(ByteBufferUtil.toBytes(tke.prevEndRow)), false, false);
-
+
check();
}
-
+
/**
* Returns a String representing this extent's entry in the Metadata table
*
@@ -129,48 +129,48 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public Text getMetadataEntry() {
return getMetadataEntry(getTableId(), getEndRow());
}
-
+
public static Text getMetadataEntry(Text tableId, Text endRow) {
return MetadataSchema.TabletsSection.getRow(tableId, endRow);
}
-
+
// constructor for loading extents from metadata rows
public KeyExtent(Text flattenedExtent, Value prevEndRow) {
decodeMetadataRow(flattenedExtent);
-
+
// decode the prev row
this.setPrevEndRow(decodePrevEndRow(prevEndRow), false, true);
-
+
check();
}
-
+
// recreates an encoded extent from a string representation
// this encoding is what is stored as the row id of the metadata table
public KeyExtent(Text flattenedExtent, Text prevEndRow) {
-
+
decodeMetadataRow(flattenedExtent);
-
+
this.setPrevEndRow(null, false, false);
if (prevEndRow != null)
this.setPrevEndRow(prevEndRow, false, true);
-
+
check();
}
-
+
/**
* Sets the extents table id
*
*/
public void setTableId(Text tId) {
-
+
if (tId == null)
throw new IllegalArgumentException("null table name not allowed");
-
+
this.textTableId = dedupeTableId(tId);
-
+
hashCode = 0;
}
-
+
/**
* Returns the extent's table id
*
@@ -178,7 +178,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public Text getTableId() {
return textTableId;
}
-
+
private void setEndRow(Text endRow, boolean check, boolean copy) {
if (endRow != null)
if (copy)
@@ -187,12 +187,12 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
this.textEndRow = endRow;
else
this.textEndRow = null;
-
+
hashCode = 0;
if (check)
check();
}
-
+
/**
* Sets this extent's end row
*
@@ -200,7 +200,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public void setEndRow(Text endRow) {
setEndRow(endRow, true, true);
}
-
+
/**
* Returns this extent's end row
*
@@ -208,7 +208,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public Text getEndRow() {
return textEndRow;
}
-
+
/**
* Return the previous extent's end row
*
@@ -216,7 +216,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public Text getPrevEndRow() {
return textPrevEndRow;
}
-
+
private void setPrevEndRow(Text prevEndRow, boolean check, boolean copy) {
if (prevEndRow != null)
if (copy)
@@ -225,12 +225,12 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
this.textPrevEndRow = prevEndRow;
else
this.textPrevEndRow = null;
-
+
hashCode = 0;
if (check)
check();
}
-
+
/**
* Sets the previous extent's end row
*
@@ -238,7 +238,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public void setPrevEndRow(Text prevEndRow) {
setPrevEndRow(prevEndRow, true, true);
}
-
+
/**
* Populates the extents data fields from a DataInput object
*
@@ -264,11 +264,11 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
} else {
setPrevEndRow((Text) null);
}
-
+
hashCode = 0;
check();
}
-
+
/**
* Writes this extent's data fields to a DataOutput object
*
@@ -289,7 +289,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
out.writeBoolean(false);
}
}
-
+
/**
* Returns a String representing the previous extent's entry in the Metadata table
*
@@ -297,13 +297,13 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public Mutation getPrevRowUpdateMutation() {
return getPrevRowUpdateMutation(this);
}
-
+
/**
* Empty start or end rows tell the method there are no start or end rows, and to use all the keyextents that are before the end row if no start row etc.
*
* @return all the key extents that the rows cover
*/
-
+
public static Collection<KeyExtent> getKeyExtentsForRange(Text startRow, Text endRow, Set<KeyExtent> kes) {
if (kes == null)
return Collections.emptyList();
@@ -340,7 +340,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
// tablet in the middle
if (startRow.getLength() == 0) {
// no start row
-
+
if (endRow.getLength() == 0) {
// no start & end row
keys.add(ckes);
@@ -361,24 +361,24 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
keys.add(ckes);
}
}
-
+
}
}
}
return keys;
}
-
+
public static Text decodePrevEndRow(Value ibw) {
Text per = null;
-
+
if (ibw.get()[0] != 0) {
per = new Text();
per.set(ibw.get(), 1, ibw.get().length - 1);
}
-
+
return per;
}
-
+
public static Value encodePrevEndRow(Text per) {
if (per == null)
return new Value(new byte[] {0});
@@ -387,31 +387,31 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
System.arraycopy(per.getBytes(), 0, b, 1, per.getLength());
return new Value(b);
}
-
+
public static Mutation getPrevRowUpdateMutation(KeyExtent ke) {
Mutation m = new Mutation(ke.getMetadataEntry());
TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.put(m, encodePrevEndRow(ke.getPrevEndRow()));
return m;
}
-
+
/**
* Compares extents based on rows
*
*/
@Override
public int compareTo(KeyExtent other) {
-
+
int result = getTableId().compareTo(other.getTableId());
if (result != 0)
return result;
-
+
if (this.getEndRow() == null) {
if (other.getEndRow() != null)
return 1;
} else {
if (other.getEndRow() == null)
return -1;
-
+
result = getEndRow().compareTo(other.getEndRow());
if (result != 0)
return result;
@@ -425,35 +425,35 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
return 1;
return this.getPrevEndRow().compareTo(other.getPrevEndRow());
}
-
+
private int hashCode = 0;
-
+
@Override
public int hashCode() {
if (hashCode != 0)
return hashCode;
-
+
int prevEndRowHash = 0;
int endRowHash = 0;
if (this.getEndRow() != null) {
endRowHash = this.getEndRow().hashCode();
}
-
+
if (this.getPrevEndRow() != null) {
prevEndRowHash = this.getPrevEndRow().hashCode();
}
-
+
hashCode = getTableId().hashCode() + endRowHash + prevEndRowHash;
return hashCode;
}
-
+
private boolean equals(Text t1, Text t2) {
if (t1 == null || t2 == null)
return t1 == t2;
-
+
return t1.equals(t2);
}
-
+
@Override
public boolean equals(Object o) {
if (o == this)
@@ -463,47 +463,47 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
KeyExtent oke = (KeyExtent) o;
return textTableId.equals(oke.textTableId) && equals(textEndRow, oke.textEndRow) && equals(textPrevEndRow, oke.textPrevEndRow);
}
-
+
@Override
public String toString() {
String endRowString;
String prevEndRowString;
String tableIdString = getTableId().toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
-
+
if (getEndRow() == null)
endRowString = "<";
else
endRowString = ";" + TextUtil.truncate(getEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
-
+
if (getPrevEndRow() == null)
prevEndRowString = "<";
else
prevEndRowString = ";" + TextUtil.truncate(getPrevEndRow()).toString().replaceAll(";", "\\\\;").replaceAll("\\\\", "\\\\\\\\");
-
+
return tableIdString + endRowString + prevEndRowString;
}
-
+
public UUID getUUID() {
try {
-
+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
-
+
// to get a unique hash it is important to encode the data
// like it is being serialized
-
+
this.write(dos);
-
+
dos.close();
-
+
return UUID.nameUUIDFromBytes(baos.toByteArray());
-
+
} catch (IOException e) {
// should not happen since we are writing to memory
throw new RuntimeException(e);
}
}
-
+
// note: this is only the encoding of the table id and the last row, not the prev row
/**
* Populates the extent's fields based on a flatted extent
@@ -512,183 +512,183 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
private void decodeMetadataRow(Text flattenedExtent) {
int semiPos = -1;
int ltPos = -1;
-
+
for (int i = 0; i < flattenedExtent.getLength(); i++) {
if (flattenedExtent.getBytes()[i] == ';' && semiPos < 0) {
// want the position of the first semicolon
semiPos = i;
}
-
+
if (flattenedExtent.getBytes()[i] == '<') {
ltPos = i;
}
}
-
+
if (semiPos < 0 && ltPos < 0) {
throw new IllegalArgumentException("Metadata row does not contain ; or < " + flattenedExtent);
}
-
+
if (semiPos < 0) {
-
+
if (ltPos != flattenedExtent.getLength() - 1) {
throw new IllegalArgumentException("< must come at end of Metadata row " + flattenedExtent);
}
-
+
Text tableId = new Text();
tableId.set(flattenedExtent.getBytes(), 0, flattenedExtent.getLength() - 1);
this.setTableId(tableId);
this.setEndRow(null, false, false);
} else {
-
+
Text tableId = new Text();
tableId.set(flattenedExtent.getBytes(), 0, semiPos);
-
+
Text endRow = new Text();
endRow.set(flattenedExtent.getBytes(), semiPos + 1, flattenedExtent.getLength() - (semiPos + 1));
-
+
this.setTableId(tableId);
-
+
this.setEndRow(endRow, false, false);
}
}
-
+
public static byte[] tableOfMetadataRow(Text row) {
KeyExtent ke = new KeyExtent();
ke.decodeMetadataRow(row);
return TextUtil.getBytes(ke.getTableId());
}
-
+
public boolean contains(final ByteSequence bsrow) {
if (bsrow == null) {
throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
}
-
+
BinaryComparable row = new BinaryComparable() {
-
+
@Override
public int getLength() {
return bsrow.length();
}
-
+
@Override
public byte[] getBytes() {
if (bsrow.isBackedByArray() && bsrow.offset() == 0)
return bsrow.getBackingArray();
-
+
return bsrow.toArray();
}
};
-
+
if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
return true;
}
return false;
}
-
+
public boolean contains(BinaryComparable row) {
if (row == null) {
throw new IllegalArgumentException("Passing null to contains is ambiguous, could be in first or last extent of table");
}
-
+
if ((this.getPrevEndRow() == null || this.getPrevEndRow().compareTo(row) < 0) && (this.getEndRow() == null || this.getEndRow().compareTo(row) >= 0)) {
return true;
}
return false;
}
-
+
public Range toDataRange() {
return new Range(getPrevEndRow(), false, getEndRow(), true);
}
-
+
public Range toMetadataRange() {
Text metadataPrevRow = new Text(getTableId());
metadataPrevRow.append(new byte[] {';'}, 0, 1);
if (getPrevEndRow() != null) {
metadataPrevRow.append(getPrevEndRow().getBytes(), 0, getPrevEndRow().getLength());
}
-
+
Range range = new Range(metadataPrevRow, getPrevEndRow() == null, getMetadataEntry(), true);
return range;
}
-
+
public static SortedSet<KeyExtent> findChildren(KeyExtent ke, SortedSet<KeyExtent> tablets) {
-
+
SortedSet<KeyExtent> children = null;
-
+
for (KeyExtent tabletKe : tablets) {
-
+
if (ke.getPrevEndRow() == tabletKe.getPrevEndRow() || ke.getPrevEndRow() != null && tabletKe.getPrevEndRow() != null
&& tabletKe.getPrevEndRow().compareTo(ke.getPrevEndRow()) == 0) {
children = new TreeSet<KeyExtent>();
}
-
+
if (children != null) {
children.add(tabletKe);
}
-
+
if (ke.getEndRow() == tabletKe.getEndRow() || ke.getEndRow() != null && tabletKe.getEndRow() != null
&& tabletKe.getEndRow().compareTo(ke.getEndRow()) == 0) {
return children;
}
}
-
+
return new TreeSet<KeyExtent>();
}
-
+
public static KeyExtent findContainingExtent(KeyExtent extent, SortedSet<KeyExtent> extents) {
-
+
KeyExtent lookupExtent = new KeyExtent(extent);
lookupExtent.setPrevEndRow((Text) null);
-
+
SortedSet<KeyExtent> tailSet = extents.tailSet(lookupExtent);
-
+
if (tailSet.isEmpty()) {
return null;
}
-
+
KeyExtent first = tailSet.first();
-
+
if (first.getTableId().compareTo(extent.getTableId()) != 0) {
return null;
}
-
+
if (first.getPrevEndRow() == null) {
return first;
}
-
+
if (extent.getPrevEndRow() == null) {
return null;
}
-
+
if (extent.getPrevEndRow().compareTo(first.getPrevEndRow()) >= 0)
return first;
return null;
}
-
+
private static boolean startsAfter(KeyExtent nke, KeyExtent ke) {
-
+
int tiCmp = ke.getTableId().compareTo(nke.getTableId());
-
+
if (tiCmp > 0) {
return true;
}
-
+
return ke.getPrevEndRow() != null && nke.getEndRow() != null && ke.getPrevEndRow().compareTo(nke.getEndRow()) >= 0;
}
-
+
private static Text rowAfterPrevRow(KeyExtent nke) {
Text row = new Text(nke.getPrevEndRow());
row.append(new byte[] {0}, 0, 1);
return row;
}
-
+
// Some duplication with TabletLocatorImpl
public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedSet<KeyExtent> extents) {
if (nke == null || extents == null || extents.isEmpty())
return Collections.emptySet();
-
+
SortedSet<KeyExtent> start;
-
+
if (nke.getPrevEndRow() != null) {
Text row = rowAfterPrevRow(nke);
KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
@@ -697,7 +697,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
start = extents.tailSet(lookupKey);
}
-
+
TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
for (KeyExtent ke : start) {
if (startsAfter(nke, ke)) {
@@ -707,20 +707,20 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
}
return result;
}
-
+
public boolean overlaps(KeyExtent other) {
SortedSet<KeyExtent> set = new TreeSet<KeyExtent>();
set.add(other);
return !findOverlapping(this, set).isEmpty();
}
-
+
// Specialization of findOverlapping(KeyExtent, SortedSet<KeyExtent> to work with SortedMap
- public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedMap<KeyExtent,? extends Object> extents) {
+ public static Set<KeyExtent> findOverlapping(KeyExtent nke, SortedMap<KeyExtent,?> extents) {
if (nke == null || extents == null || extents.isEmpty())
return Collections.emptySet();
-
- SortedMap<KeyExtent,? extends Object> start;
-
+
+ SortedMap<KeyExtent,?> start;
+
if (nke.getPrevEndRow() != null) {
Text row = rowAfterPrevRow(nke);
KeyExtent lookupKey = new KeyExtent(nke.getTableId(), row, null);
@@ -729,9 +729,9 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
KeyExtent lookupKey = new KeyExtent(nke.getTableId(), new Text(), null);
start = extents.tailMap(lookupKey);
}
-
+
TreeSet<KeyExtent> result = new TreeSet<KeyExtent>();
- for (Entry<KeyExtent,? extends Object> entry : start.entrySet()) {
+ for (Entry<KeyExtent,?> entry : start.entrySet()) {
KeyExtent ke = entry.getKey();
if (startsAfter(nke, ke)) {
break;
@@ -740,36 +740,36 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
}
return result;
}
-
+
public static Text getMetadataEntry(KeyExtent extent) {
return getMetadataEntry(extent.getTableId(), extent.getEndRow());
}
-
+
public TKeyExtent toThrift() {
return new TKeyExtent(TextUtil.getByteBuffer(textTableId), textEndRow == null ? null : TextUtil.getByteBuffer(textEndRow), textPrevEndRow == null ? null
: TextUtil.getByteBuffer(textPrevEndRow));
}
-
+
public boolean isPreviousExtent(KeyExtent prevExtent) {
if (prevExtent == null)
return getPrevEndRow() == null;
-
+
if (!prevExtent.getTableId().equals(getTableId()))
throw new IllegalArgumentException("Cannot compare accross tables " + prevExtent + " " + this);
-
+
if (prevExtent.getEndRow() == null)
return false;
-
+
if (getPrevEndRow() == null)
return false;
-
+
return prevExtent.getEndRow().equals(getPrevEndRow());
}
-
+
public boolean isMeta() {
return getTableId().toString().equals(MetadataTable.ID) || isRootTablet();
}
-
+
public boolean isRootTablet() {
return getTableId().toString().equals(RootTable.ID);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/core/src/main/java/org/apache/accumulo/core/util/ArgumentChecker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/ArgumentChecker.java b/core/src/main/java/org/apache/accumulo/core/util/ArgumentChecker.java
index 8966299..20a1373 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/ArgumentChecker.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ArgumentChecker.java
@@ -25,43 +25,43 @@ package org.apache.accumulo.core.util;
*/
public class ArgumentChecker {
private static final String NULL_ARG_MSG = "argument was null";
-
+
public static final void notNull(final Object arg1) {
if (arg1 == null)
throw new IllegalArgumentException(NULL_ARG_MSG + ":Is null- arg1? " + (arg1 == null));
}
-
+
public static final void notNull(final Object arg1, final Object arg2) {
if (arg1 == null || arg2 == null)
throw new IllegalArgumentException(NULL_ARG_MSG + ":Is null- arg1? " + (arg1 == null) + " arg2? " + (arg2 == null));
}
-
+
public static final void notNull(final Object arg1, final Object arg2, final Object arg3) {
if (arg1 == null || arg2 == null || arg3 == null)
throw new IllegalArgumentException(NULL_ARG_MSG + ":Is null- arg1? " + (arg1 == null) + " arg2? " + (arg2 == null) + " arg3? " + (arg3 == null));
}
-
+
public static final void notNull(final Object arg1, final Object arg2, final Object arg3, final Object arg4) {
if (arg1 == null || arg2 == null || arg3 == null || arg4 == null)
throw new IllegalArgumentException(NULL_ARG_MSG + ":Is null- arg1? " + (arg1 == null) + " arg2? " + (arg2 == null) + " arg3? " + (arg3 == null)
+ " arg4? " + (arg4 == null));
}
-
+
public static final void notNull(final Object[] args) {
if (args == null)
throw new IllegalArgumentException(NULL_ARG_MSG + ":arg array is null");
-
+
for (int i = 0; i < args.length; i++)
if (args[i] == null)
throw new IllegalArgumentException(NULL_ARG_MSG + ":arg" + i + " is null");
}
-
+
public static final void strictlyPositive(final int i) {
if (i <= 0)
throw new IllegalArgumentException("integer should be > 0, was " + i);
}
-
- public static final void notEmpty(Iterable<? extends Object> arg) {
+
+ public static final void notEmpty(Iterable<?> arg) {
if (!arg.iterator().hasNext())
throw new IllegalArgumentException("Argument should not be empty");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 2822c50..e83453e 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@ -36,8 +36,8 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.RegExFilter;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
-import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
@@ -48,6 +48,8 @@ import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
public class AccumuloInputFormatTest {
@@ -56,6 +58,18 @@ public class AccumuloInputFormatTest {
private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+ private JobConf job;
+
+ @BeforeClass
+ public static void setupClass() {
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+ }
+
+ @Before
+ public void createJob() {
+ job = new JobConf();
+ }
+
/**
* Check that the iterator configuration is getting stored in the Job conf correctly.
*
@@ -63,8 +77,6 @@ public class AccumuloInputFormatTest {
*/
@Test
public void testSetIterator() throws IOException {
- JobConf job = new JobConf();
-
IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
AccumuloInputFormat.addIterator(job, is);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -75,8 +87,6 @@ public class AccumuloInputFormatTest {
@Test
public void testAddIterator() throws IOException {
- JobConf job = new JobConf();
-
AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
@@ -123,7 +133,6 @@ public class AccumuloInputFormatTest {
String value = "comma,delimited,value";
IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
someSetting.addOption(key, value);
- JobConf job = new JobConf();
AccumuloInputFormat.addIterator(job, someSetting);
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
@@ -151,8 +160,6 @@ public class AccumuloInputFormatTest {
*/
@Test
public void testGetIteratorSettings() throws IOException {
- JobConf job = new JobConf();
-
AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
@@ -182,8 +189,6 @@ public class AccumuloInputFormatTest {
@Test
public void testSetRegex() throws IOException {
- JobConf job = new JobConf();
-
String regex = ">\"*%<>\'\\";
IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
@@ -259,8 +264,8 @@ public class AccumuloInputFormatTest {
return JobClient.runJob(job).isSuccessful() ? 0 : 1;
}
- public static void main(String[] args) throws Exception {
- assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+ public static void main(String... args) throws Exception {
+ assertEquals(0, ToolRunner.run(new Configuration(), new MRTester(), args));
}
}
@@ -277,7 +282,7 @@ public class AccumuloInputFormatTest {
}
bw.close();
- MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+ MRTester.main("root", "", TEST_TABLE_1);
assertNull(e1);
assertNull(e2);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
index ad03551..50a9fbe 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockNamespacesTest.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.core.client.mock;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.File;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Map.Entry;
@@ -52,7 +53,7 @@ import org.junit.rules.TemporaryFolder;
public class MockNamespacesTest {
Random random = new Random();
- public static TemporaryFolder folder = new TemporaryFolder();
+ public static TemporaryFolder folder = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
/**
* This test creates a table without specifying a namespace. In this case, it puts the table into the default namespace.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
index ce91da6..689f07c 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/RandomBatchWriter.java
@@ -46,7 +46,7 @@ import com.beust.jcommander.Parameter;
* qualifiers will be "1". The values will be random byte arrays of a specified size.
*/
public class RandomBatchWriter {
-
+
/**
* Creates a random byte array of specified size using the specified seed.
*
@@ -59,17 +59,17 @@ public class RandomBatchWriter {
public static byte[] createValue(long rowid, int dataSize) {
Random r = new Random(rowid);
byte value[] = new byte[dataSize];
-
+
r.nextBytes(value);
-
+
// transform to printable chars
for (int j = 0; j < value.length; j++) {
value[j] = (byte) (((0xff & value[j]) % 92) + ' ');
}
-
+
return value;
}
-
+
/**
* Creates a mutation on a specified row with column family "foo", column qualifier "1", specified visibility, and a random value of specified size.
*
@@ -83,33 +83,33 @@ public class RandomBatchWriter {
*/
public static Mutation createMutation(long rowid, int dataSize, ColumnVisibility visibility) {
Text row = new Text(String.format("row_%010d", rowid));
-
+
Mutation m = new Mutation(row);
-
+
// create a random value that is a function of the
// row id for verification purposes
byte value[] = createValue(rowid, dataSize);
-
+
m.put(new Text("foo"), new Text("1"), visibility, new Value(value));
-
+
return m;
}
-
+
static class Opts extends ClientOnRequiredTable {
- @Parameter(names="--num", required=true)
+ @Parameter(names = "--num", required = true)
int num = 0;
- @Parameter(names="--min")
+ @Parameter(names = "--min")
long min = 0;
- @Parameter(names="--max")
+ @Parameter(names = "--max")
long max = Long.MAX_VALUE;
- @Parameter(names="--size", required=true, description="size of the value to write")
+ @Parameter(names = "--size", required = true, description = "size of the value to write")
int size = 0;
- @Parameter(names="--vis", converter=VisibilityConverter.class)
+ @Parameter(names = "--vis", converter = VisibilityConverter.class)
ColumnVisibility visiblity = new ColumnVisibility("");
- @Parameter(names="--seed", description="seed for pseudo-random number generator")
+ @Parameter(names = "--seed", description = "seed for pseudo-random number generator")
Long seed = null;
}
-
+
/**
* Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
*
@@ -122,7 +122,11 @@ public class RandomBatchWriter {
BatchWriterOpts bwOpts = new BatchWriterOpts();
opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
if ((opts.max - opts.min) < opts.num) {
- System.err.println(String.format("You must specify a min and a max that allow for at least num possible values. For example, you requested %d rows, but a min of %d and a max of %d only allows for %d rows.", opts.num, opts.min, opts.max, (opts.max - opts.min)));
+ System.err
+ .println(String
+ .format(
+ "You must specify a min and a max that allow for at least num possible values. For example, you requested %d rows, but a min of %d and a max of %d (exclusive), which only allows for %d rows.",
+ opts.num, opts.min, opts.max, (opts.max - opts.min)));
System.exit(1);
}
Random r;
@@ -133,10 +137,10 @@ public class RandomBatchWriter {
}
Connector connector = opts.getConnector();
BatchWriter bw = connector.createBatchWriter(opts.tableName, bwOpts.getBatchWriterConfig());
-
+
// reuse the ColumnVisibility object to improve performance
ColumnVisibility cv = opts.visiblity;
-
+
// Generate num unique row ids in the given range
HashSet<Long> rowids = new HashSet<Long>(opts.num);
while (rowids.size() < opts.num) {
@@ -146,7 +150,7 @@ public class RandomBatchWriter {
Mutation m = createMutation(rowid, opts.size, cv);
bw.addMutation(m);
}
-
+
try {
bw.close();
} catch (MutationsRejectedException e) {
@@ -162,7 +166,7 @@ public class RandomBatchWriter {
}
System.err.println("ERROR : Not authorized to write to tables : " + tables);
}
-
+
if (e.getConstraintViolationSummaries().size() > 0) {
System.err.println("ERROR : Constraint violations occurred : " + e.getConstraintViolationSummaries());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
index dab1e10..3f57119 100644
--- a/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
+++ b/examples/simple/src/test/java/org/apache/accumulo/examples/simple/filedata/ChunkInputFormatTest.java
@@ -16,14 +16,17 @@
*/
package org.apache.accumulo.examples.simple.filedata;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
-import junit.framework.TestCase;
-
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
@@ -42,8 +45,11 @@ import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ChunkInputFormatTest {
-public class ChunkInputFormatTest extends TestCase {
private static AssertionError e0 = null;
private static AssertionError e1 = null;
private static AssertionError e2 = null;
@@ -54,7 +60,10 @@ public class ChunkInputFormatTest extends TestCase {
private static List<Entry<Key,Value>> data;
private static List<Entry<Key,Value>> baddata;
- {
+ @BeforeClass
+ public static void setupClass() {
+ System.setProperty("hadoop.tmp.dir", System.getProperty("user.dir") + "/target/hadoop-tmp");
+
data = new ArrayList<Entry<Key,Value>>();
ChunkInputStreamTest.addData(data, "a", "refs", "ida\0ext", "A&B", "ext");
ChunkInputStreamTest.addData(data, "a", "refs", "ida\0name", "A&B", "name");
@@ -219,11 +228,12 @@ public class ChunkInputFormatTest extends TestCase {
return job.isSuccessful() ? 0 : 1;
}
- public static int main(String[] args) throws Exception {
+ public static int main(String... args) throws Exception {
return ToolRunner.run(new Configuration(), new CIFTester(), args);
}
}
+ @Test
public void test() throws Exception {
MockInstance instance = new MockInstance("instance1");
Connector conn = instance.getConnector("root", new PasswordToken(""));
@@ -238,11 +248,12 @@ public class ChunkInputFormatTest extends TestCase {
}
bw.close();
- assertEquals(0, CIFTester.main(new String[] {"instance1", "root", "", "test", CIFTester.TestMapper.class.getName()}));
+ assertEquals(0, CIFTester.main("instance1", "root", "", "test", CIFTester.TestMapper.class.getName()));
assertNull(e1);
assertNull(e2);
}
+ @Test
public void testErrorOnNextWithoutClose() throws Exception {
MockInstance instance = new MockInstance("instance2");
Connector conn = instance.getConnector("root", new PasswordToken(""));
@@ -257,12 +268,13 @@ public class ChunkInputFormatTest extends TestCase {
}
bw.close();
- assertEquals(1, CIFTester.main(new String[] {"instance2", "root", "", "test", CIFTester.TestNoClose.class.getName()}));
+ assertEquals(1, CIFTester.main("instance2", "root", "", "test", CIFTester.TestNoClose.class.getName()));
assertNull(e1);
assertNull(e2);
assertNotNull(e3);
}
+ @Test
public void testInfoWithoutChunks() throws Exception {
MockInstance instance = new MockInstance("instance3");
Connector conn = instance.getConnector("root", new PasswordToken(""));
@@ -276,7 +288,7 @@ public class ChunkInputFormatTest extends TestCase {
}
bw.close();
- assertEquals(0, CIFTester.main(new String[] {"instance3", "root", "", "test", CIFTester.TestBadData.class.getName()}));
+ assertEquals(0, CIFTester.main("instance3", "root", "", "test", CIFTester.TestBadData.class.getName()));
assertNull(e0);
assertNull(e1);
assertNull(e2);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 09f60c2..2f8f433 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -150,8 +150,16 @@ public class MiniAccumuloCluster {
private MiniDFSCluster miniDFS = null;
private List<Process> cleanup = new ArrayList<Process>();
- public Process exec(Class<? extends Object> clazz, String... args) throws IOException {
- Process proc = exec(clazz, Collections.singletonList("-Xmx" + config.getDefaultMemory()), args);
+ public Process exec(Class<?> clazz, String... args) throws IOException {
+ return exec(clazz, null, args);
+ }
+
+ public Process exec(Class<?> clazz, List<String> jvmArgs, String... args) throws IOException {
+ ArrayList<String> jvmArgs2 = new ArrayList<String>(1 + (jvmArgs == null ? 0 : jvmArgs.size()));
+ jvmArgs2.add("-Xmx" + config.getDefaultMemory());
+ if (jvmArgs != null)
+ jvmArgs2.addAll(jvmArgs);
+ Process proc = _exec(clazz, jvmArgs2, args);
cleanup.add(proc);
return proc;
}
@@ -226,7 +234,7 @@ public class MiniAccumuloCluster {
}
}
- private Process exec(Class<? extends Object> clazz, List<String> extraJvmOpts, String... args) throws IOException {
+ private Process _exec(Class<?> clazz, List<String> extraJvmOpts, String... args) throws IOException {
String javaHome = System.getProperty("java.home");
String javaBin = javaHome + File.separator + "bin" + File.separator + "java";
String classpath = getClasspath();
@@ -275,7 +283,7 @@ public class MiniAccumuloCluster {
return process;
}
- private Process exec(Class<? extends Object> clazz, ServerType serverType, String... args) throws IOException {
+ private Process _exec(Class<?> clazz, ServerType serverType, String... args) throws IOException {
List<String> jvmOpts = new ArrayList<String>();
jvmOpts.add("-Xmx" + config.getMemory(serverType));
@@ -285,7 +293,7 @@ public class MiniAccumuloCluster {
jvmOpts.addAll(buildRemoteDebugParams(port));
debugPorts.add(new Pair<ServerType,Integer>(serverType, port));
}
- return exec(clazz, jvmOpts, args);
+ return _exec(clazz, jvmOpts, args);
}
/**
@@ -425,7 +433,7 @@ public class MiniAccumuloCluster {
}
if (zooKeeperProcess == null) {
- zooKeeperProcess = exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
+ zooKeeperProcess = _exec(ZooKeeperServerMain.class, ServerType.ZOOKEEPER, zooCfgFile.getAbsolutePath());
}
if (!initialized) {
@@ -456,7 +464,7 @@ public class MiniAccumuloCluster {
}
synchronized (tabletServerProcesses) {
for (int i = tabletServerProcesses.size(); i < config.getNumTservers(); i++) {
- tabletServerProcesses.add(exec(TabletServer.class, ServerType.TABLET_SERVER));
+ tabletServerProcesses.add(_exec(TabletServer.class, ServerType.TABLET_SERVER));
}
}
int ret = 0;
@@ -470,10 +478,10 @@ public class MiniAccumuloCluster {
throw new RuntimeException("Could not set master goal state, process returned " + ret + ". Check the logs in " + config.getLogDir() + " for errors.");
}
if (masterProcess == null) {
- masterProcess = exec(Master.class, ServerType.MASTER);
+ masterProcess = _exec(Master.class, ServerType.MASTER);
}
if (config.shouldRunGC()) {
- gcProcess = exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
+ gcProcess = _exec(SimpleGarbageCollector.class, ServerType.GARBAGE_COLLECTOR);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java b/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
index 7ec9fd2..eca7dfe 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/AuditedSecurityOperation.java
@@ -54,19 +54,19 @@ import org.apache.log4j.Logger;
*
*/
public class AuditedSecurityOperation extends SecurityOperation {
-
+
public static final String AUDITLOG = "Audit";
public static final Logger audit = Logger.getLogger(AUDITLOG);
-
+
public AuditedSecurityOperation(Authorizor author, Authenticator authent, PermissionHandler pm, String instanceId) {
super(author, authent, pm, instanceId);
}
-
+
public static synchronized SecurityOperation getInstance() {
String instanceId = HdfsZooInstance.getInstance().getInstanceID();
return getInstance(instanceId, false);
}
-
+
public static synchronized SecurityOperation getInstance(String instanceId, boolean initialize) {
if (instance == null) {
instance = new AuditedSecurityOperation(getAuthorizor(instanceId, initialize), getAuthenticator(instanceId, initialize), getPermHandler(instanceId,
@@ -74,7 +74,7 @@ public class AuditedSecurityOperation extends SecurityOperation {
}
return instance;
}
-
+
private static String getTableName(String tableId) {
try {
return Tables.getTableName(HdfsZooInstance.getInstance(), tableId);
@@ -82,7 +82,7 @@ public class AuditedSecurityOperation extends SecurityOperation {
return "Unknown Table with ID " + tableId;
}
}
-
+
public static StringBuilder getAuthString(List<ByteBuffer> authorizations) {
StringBuilder auths = new StringBuilder();
for (ByteBuffer bb : authorizations) {
@@ -90,16 +90,16 @@ public class AuditedSecurityOperation extends SecurityOperation {
}
return auths;
}
-
+
private static boolean shouldAudit(TCredentials credentials, String tableId) {
return !tableId.equals(MetadataTable.ID) && shouldAudit(credentials);
}
-
+
// Is INFO the right level to check? Do we even need that check?
private static boolean shouldAudit(TCredentials credentials) {
return !SystemCredentials.get().getToken().getClass().getName().equals(credentials.getTokenClassName());
}
-
+
/*
* Three auditing methods try to capture the 4 states we might have here. audit is in response to a thrown exception, the operation failed (perhaps due to
* insufficient privs, or some other reason) audit(credentials, template, args) is a successful operation audit(credentials, permitted, template, args) is a
@@ -109,24 +109,24 @@ public class AuditedSecurityOperation extends SecurityOperation {
private void audit(TCredentials credentials, ThriftSecurityException ex, String template, Object... args) {
audit.warn("operation: failed; user: " + credentials.getPrincipal() + "; " + String.format(template, args) + "; exception: " + ex.toString());
}
-
+
private void audit(TCredentials credentials, String template, Object... args) {
if (shouldAudit(credentials)) {
audit.info("operation: success; user: " + credentials.getPrincipal() + ": " + String.format(template, args));
}
}
-
+
private void audit(TCredentials credentials, boolean permitted, String template, Object... args) {
if (shouldAudit(credentials)) {
String prefix = permitted ? "permitted" : "denied";
audit.info("operation: " + prefix + "; user: " + credentials.getPrincipal() + "; " + String.format(template, args));
}
}
-
+
public static final String CAN_SCAN_AUDIT_TEMPLATE = "action: scan; targetTable: %s; authorizations: %s; range: %s; columns: %s; iterators: %s; iteratorOptions: %s;";
private static final int MAX_ELEMENTS_TO_LOG = 10;
-
- private static List<String> truncate(Collection<? extends Object> list) {
+
+ private static List<String> truncate(Collection<?> list) {
List<String> result = new ArrayList<String>();
int i = 0;
for (Object obj : list) {
@@ -138,7 +138,7 @@ public class AuditedSecurityOperation extends SecurityOperation {
}
return result;
}
-
+
@Override
public boolean canScan(TCredentials credentials, String tableId, TRange range, List<TColumn> columns, List<IterInfo> ssiList,
Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
@@ -146,11 +146,11 @@ public class AuditedSecurityOperation extends SecurityOperation {
Range convertedRange = new Range(range);
List<String> convertedColumns = truncate(Translator.translate(columns, new Translator.TColumnTranslator()));
String tableName = getTableName(tableId);
-
+
try {
boolean canScan = super.canScan(credentials, tableId);
audit(credentials, canScan, CAN_SCAN_AUDIT_TEMPLATE, tableName, getAuthString(authorizations), convertedRange, convertedColumns, ssiList, ssio);
-
+
return canScan;
} catch (ThriftSecurityException ex) {
audit(credentials, ex, CAN_SCAN_AUDIT_TEMPLATE, getAuthString(authorizations), tableId, convertedRange, convertedColumns, ssiList, ssio);
@@ -160,9 +160,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
return super.canScan(credentials, tableId);
}
}
-
+
public static final String CAN_SCAN_BATCH_AUDIT_TEMPLATE = "action: scan; targetTable: %s; authorizations: %s; range: %s; columns: %s; iterators: %s; iteratorOptions: %s;";
-
+
@Override
public boolean canScan(TCredentials credentials, String tableId, Map<TKeyExtent,List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList,
Map<String,Map<String,String>> ssio, List<ByteBuffer> authorizations) throws ThriftSecurityException {
@@ -170,17 +170,17 @@ public class AuditedSecurityOperation extends SecurityOperation {
@SuppressWarnings({"unchecked", "rawtypes"})
Map<KeyExtent,List<Range>> convertedBatch = Translator.translate(tbatch, new Translator.TKeyExtentTranslator(), new Translator.ListTranslator(
new Translator.TRangeTranslator()));
- Map<KeyExtent, List<String>> truncated = new HashMap<KeyExtent, List<String>>();
+ Map<KeyExtent,List<String>> truncated = new HashMap<KeyExtent,List<String>>();
for (Entry<KeyExtent,List<Range>> entry : convertedBatch.entrySet()) {
- truncated.put(entry.getKey(), truncate(entry.getValue()));
+ truncated.put(entry.getKey(), truncate(entry.getValue()));
}
List<Column> convertedColumns = Translator.translate(tcolumns, new Translator.TColumnTranslator());
String tableName = getTableName(tableId);
-
+
try {
boolean canScan = super.canScan(credentials, tableId);
audit(credentials, canScan, CAN_SCAN_BATCH_AUDIT_TEMPLATE, tableName, getAuthString(authorizations), truncated, convertedColumns, ssiList, ssio);
-
+
return canScan;
} catch (ThriftSecurityException ex) {
audit(credentials, ex, CAN_SCAN_BATCH_AUDIT_TEMPLATE, getAuthString(authorizations), tableId, truncated, convertedColumns, ssiList, ssio);
@@ -190,9 +190,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
return super.canScan(credentials, tableId);
}
}
-
+
public static final String CHANGE_AUTHORIZATIONS_AUDIT_TEMPLATE = "action: changeAuthorizations; targetUser: %s; authorizations: %s";
-
+
@Override
public void changeAuthorizations(TCredentials credentials, String user, Authorizations authorizations) throws ThriftSecurityException {
try {
@@ -203,9 +203,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CHANGE_PASSWORD_AUDIT_TEMPLATE = "action: changePassword; targetUser: %s;";
-
+
@Override
public void changePassword(TCredentials credentials, Credentials newInfo) throws ThriftSecurityException {
try {
@@ -216,9 +216,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CREATE_USER_AUDIT_TEMPLATE = "action: createUser; targetUser: %s; Authorizations: %s;";
-
+
@Override
public void createUser(TCredentials credentials, Credentials newUser, Authorizations authorizations) throws ThriftSecurityException {
try {
@@ -229,9 +229,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_CREATE_TABLE_AUDIT_TEMPLATE = "action: createTable; targetTable: %s;";
-
+
@Override
public boolean canCreateTable(TCredentials c, String tableName) throws ThriftSecurityException {
try {
@@ -243,9 +243,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_DELETE_TABLE_AUDIT_TEMPLATE = "action: deleteTable; targetTable: %s;";
-
+
@Override
public boolean canDeleteTable(TCredentials c, String tableId) throws ThriftSecurityException {
String tableName = getTableName(tableId);
@@ -258,9 +258,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_RENAME_TABLE_AUDIT_TEMPLATE = "action: renameTable; targetTable: %s; newTableName: %s;";
-
+
@Override
public boolean canRenameTable(TCredentials c, String tableId, String oldTableName, String newTableName) throws ThriftSecurityException {
try {
@@ -272,9 +272,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_CLONE_TABLE_AUDIT_TEMPLATE = "action: cloneTable; targetTable: %s; newTableName: %s";
-
+
@Override
public boolean canCloneTable(TCredentials c, String tableId, String tableName) throws ThriftSecurityException {
String oldTableName = getTableName(tableId);
@@ -287,9 +287,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_DELETE_RANGE_AUDIT_TEMPLATE = "action: deleteData; targetTable: %s; startRange: %s; endRange: %s;";
-
+
@Override
public boolean canDeleteRange(TCredentials c, String tableId, String tableName, Text startRow, Text endRow) throws ThriftSecurityException {
try {
@@ -301,9 +301,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_BULK_IMPORT_AUDIT_TEMPLATE = "action: bulkImport; targetTable: %s; dataDir: %s; failDir: %s;";
-
+
@Override
public boolean canBulkImport(TCredentials c, String tableId, String tableName, String dir, String failDir) throws ThriftSecurityException {
try {
@@ -315,12 +315,12 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_IMPORT_AUDIT_TEMPLATE = "action: import; targetTable: %s; dataDir: %s;";
-
+
@Override
public boolean canImport(TCredentials credentials, String tableName, String importDir) throws ThriftSecurityException {
-
+
try {
boolean result = super.canImport(credentials, tableName, importDir);
audit(credentials, result, CAN_IMPORT_AUDIT_TEMPLATE, tableName, importDir);
@@ -330,12 +330,12 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_EXPORT_AUDIT_TEMPLATE = "action: export; targetTable: %s; dataDir: %s;";
-
+
@Override
public boolean canExport(TCredentials credentials, String tableId, String tableName, String exportDir) throws ThriftSecurityException {
-
+
try {
boolean result = super.canExport(credentials, tableId, tableName, exportDir);
audit(credentials, result, CAN_EXPORT_AUDIT_TEMPLATE, tableName, exportDir);
@@ -345,9 +345,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String DROP_USER_AUDIT_TEMPLATE = "action: dropUser; targetUser: %s;";
-
+
@Override
public void dropUser(TCredentials credentials, String user) throws ThriftSecurityException {
try {
@@ -358,9 +358,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String GRANT_SYSTEM_PERMISSION_AUDIT_TEMPLATE = "action: grantSystemPermission; permission: %s; targetUser: %s;";
-
+
@Override
public void grantSystemPermission(TCredentials credentials, String user, SystemPermission permission) throws ThriftSecurityException {
try {
@@ -371,9 +371,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String GRANT_TABLE_PERMISSION_AUDIT_TEMPLATE = "action: grantTablePermission; permission: %s; targetTable: %s; targetUser: %s;";
-
+
@Override
public void grantTablePermission(TCredentials credentials, String user, String tableId, TablePermission permission) throws ThriftSecurityException {
String tableName = getTableName(tableId);
@@ -385,12 +385,12 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String REVOKE_SYSTEM_PERMISSION_AUDIT_TEMPLATE = "action: revokeSystemPermission; permission: %s; targetUser: %s;";
-
+
@Override
public void revokeSystemPermission(TCredentials credentials, String user, SystemPermission permission) throws ThriftSecurityException {
-
+
try {
super.revokeSystemPermission(credentials, user, permission);
audit(credentials, REVOKE_SYSTEM_PERMISSION_AUDIT_TEMPLATE, permission, user);
@@ -399,9 +399,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String REVOKE_TABLE_PERMISSION_AUDIT_TEMPLATE = "action: revokeTablePermission; permission: %s; targetTable: %s; targetUser: %s;";
-
+
@Override
public void revokeTablePermission(TCredentials credentials, String user, String tableId, TablePermission permission) throws ThriftSecurityException {
String tableName = getTableName(tableId);
@@ -413,9 +413,9 @@ public class AuditedSecurityOperation extends SecurityOperation {
throw ex;
}
}
-
+
public static final String CAN_ONLINE_OFFLINE_TABLE_AUDIT_TEMPLATE = "action: %s; targetTable: %s;";
-
+
@Override
public boolean canOnlineOfflineTable(TCredentials credentials, String tableId, TableOperation op) throws ThriftSecurityException {
String tableName = getTableName(tableId);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
index c03aa17..f1ceb3b 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java
@@ -26,12 +26,14 @@ import java.io.OutputStream;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -39,22 +41,24 @@ public class TestUpgradePathForWALogs {
private static final String WALOG_FROM_15 = "/walog-from-15.walog";
private static final String WALOG_FROM_16 = "/walog-from-16.walog";
+ private static File testDir;
VolumeManager fs;
- TemporaryFolder root;
+ @BeforeClass
+ public static void createTestDirectory() {
+ File baseDir = new File(System.getProperty("user.dir") + "/target/upgrade-tests");
+ baseDir.mkdirs();
+ testDir = new File(baseDir, TestUpgradePathForWALogs.class.getName());
+ FileUtils.deleteQuietly(testDir);
+ testDir.mkdir();
+ }
+
+ @Rule
+ public TemporaryFolder root = new TemporaryFolder(testDir);
@Before
public void setUp() throws Exception {
- File tempFile = File.createTempFile("TestUpgradePathForWALogs", "");
- String tempDirName = tempFile.getAbsolutePath() + "Dir";
- tempFile.delete();
-
- File tempDir = new File(tempDirName);
- tempDir.mkdirs();
-
- root = new TemporaryFolder(new File(tempDirName));
-
// quiet log messages about compress.CodecPool
Logger.getRootLogger().setLevel(Level.ERROR);
fs = VolumeManagerImpl.getLocal();
@@ -63,20 +67,6 @@ public class TestUpgradePathForWALogs {
Path manyMapsPath = new Path("file://" + path + "/manyMaps");
fs.mkdirs(manyMapsPath);
fs.create(new Path(manyMapsPath, "finished")).close();
- // FileSystem ns = fs.getDefaultVolume();
- // Writer writer = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class);
- // BytesWritable value = new BytesWritable("someValue".getBytes());
- // for (int i = 1; i < 1000; i += 2) {
- // writer.append(new IntWritable(i), value);
- // }
- // writer.close();
- // writer = new Writer(ns.getConf(), ns, new Path(root, "even").toString(), IntWritable.class, BytesWritable.class);
- // for (int i = 0; i < 1000; i += 2) {
- // if (i == 10)
- // continue;
- // writer.append(new IntWritable(i), value);
- // }
- // writer.close();
}
@Test
@@ -147,8 +137,4 @@ public class TestUpgradePathForWALogs {
}
}
- @After
- public void tearDown() throws Exception {
- // root.delete();
- }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c1fbeac5/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterIT.java b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterIT.java
new file mode 100644
index 0000000..427b12f
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/MultiTableBatchWriterIT.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.MultiTableBatchWriterImpl;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.test.functional.SimpleMacIT;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class MultiTableBatchWriterIT extends SimpleMacIT {
+
+ private Connector connector;
+ private MultiTableBatchWriter mtbw;
+
+ @Before
+ public void setUpArgs() throws AccumuloException, AccumuloSecurityException {
+ connector = getConnector();
+ mtbw = getMultiTableBatchWriter(60);
+ }
+
+ public MultiTableBatchWriter getMultiTableBatchWriter(long cacheTimeoutInSeconds) {
+ return new MultiTableBatchWriterImpl(connector.getInstance(), new Credentials("root", new PasswordToken(getStaticCluster().getConfig().getRootPassword())),
+ new BatchWriterConfig(), cacheTimeoutInSeconds, TimeUnit.SECONDS);
+ }
+
+ @Test
+ public void testTableRenameDataValidation() throws Exception {
+
+ try {
+ final String table1 = "testTableRenameDataValidation_table1", table2 = "testTableRenameDataValidation_table2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+
+ bw1.addMutation(m1);
+
+ tops.rename(table1, table2);
+ tops.create(table1);
+
+ BatchWriter bw2 = mtbw.getBatchWriter(table1);
+
+ Mutation m2 = new Mutation("bar");
+ m2.put("col1", "", "val1");
+
+ bw1.addMutation(m2);
+ bw2.addMutation(m2);
+
+ mtbw.close();
+
+ Map<Entry<String,String>,String> table1Expectations = new HashMap<Entry<String,String>,String>();
+ table1Expectations.put(Maps.immutableEntry("bar", "col1"), "val1");
+
+ Map<Entry<String,String>,String> table2Expectations = new HashMap<Entry<String,String>,String>();
+ table2Expectations.put(Maps.immutableEntry("foo", "col1"), "val1");
+ table2Expectations.put(Maps.immutableEntry("bar", "col1"), "val1");
+
+ Scanner s = connector.createScanner(table1, new Authorizations());
+ s.setRange(new Range());
+ Map<Entry<String,String>,String> actual = new HashMap<Entry<String,String>,String>();
+ for (Entry<Key,Value> entry : s) {
+ actual.put(Maps.immutableEntry(entry.getKey().getRow().toString(), entry.getKey().getColumnFamily().toString()), entry.getValue().toString());
+ }
+
+ Assert.assertEquals("Differing results for " + table1, table1Expectations, actual);
+
+ s = connector.createScanner(table2, new Authorizations());
+ s.setRange(new Range());
+ actual = new HashMap<Entry<String,String>,String>();
+ for (Entry<Key,Value> entry : s) {
+ actual.put(Maps.immutableEntry(entry.getKey().getRow().toString(), entry.getKey().getColumnFamily().toString()), entry.getValue().toString());
+ }
+
+ Assert.assertEquals("Differing results for " + table2, table2Expectations, actual);
+
+ } finally {
+ if (null != mtbw) {
+ mtbw.close();
+ }
+ }
+ }
+
+ @Test
+ public void testTableRenameSameWriters() throws Exception {
+
+ try {
+ final String table1 = "testTableRenameSameWriters_table1", table2 = "testTableRenameSameWriters_table2";
+ final String newTable1 = "testTableRenameSameWriters_newTable1", newTable2 = "testTableRenameSameWriters_newTable2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ tops.rename(table1, newTable1);
+ tops.rename(table2, newTable2);
+
+ Mutation m2 = new Mutation("bar");
+ m2.put("col1", "", "val1");
+ m2.put("col2", "", "val2");
+
+ bw1.addMutation(m2);
+ bw2.addMutation(m2);
+
+ mtbw.close();
+
+ Map<Entry<String,String>,String> expectations = new HashMap<Entry<String,String>,String>();
+ expectations.put(Maps.immutableEntry("foo", "col1"), "val1");
+ expectations.put(Maps.immutableEntry("foo", "col2"), "val2");
+ expectations.put(Maps.immutableEntry("bar", "col1"), "val1");
+ expectations.put(Maps.immutableEntry("bar", "col2"), "val2");
+
+ for (String table : Arrays.asList(newTable1, newTable2)) {
+ Scanner s = connector.createScanner(table, new Authorizations());
+ s.setRange(new Range());
+ Map<Entry<String,String>,String> actual = new HashMap<Entry<String,String>,String>();
+ for (Entry<Key,Value> entry : s) {
+ actual.put(Maps.immutableEntry(entry.getKey().getRow().toString(), entry.getKey().getColumnFamily().toString()), entry.getValue().toString());
+ }
+
+ Assert.assertEquals("Differing results for " + table, expectations, actual);
+ }
+ } finally {
+ if (null != mtbw) {
+ mtbw.close();
+ }
+ }
+ }
+
+ @Test
+ public void testTableRenameNewWriters() throws Exception {
+
+ try {
+ final String table1 = "testTableRenameNewWriters_table1", table2 = "testTableRenameNewWriters_table2";
+ final String newTable1 = "testTableRenameNewWriters_newTable1", newTable2 = "testTableRenameNewWriters_newTable2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ tops.rename(table1, newTable1);
+
+ // MTBW is still caching this name to the correct table, but we should invalidate its cache
+ // after seeing the rename
+ try {
+ bw1 = mtbw.getBatchWriter(table1);
+ Assert.fail("Should not be able to find this table");
+ } catch (TableNotFoundException e) {
+ // pass
+ }
+
+ tops.rename(table2, newTable2);
+
+ try {
+ bw2 = mtbw.getBatchWriter(table2);
+ Assert.fail("Should not be able to find this table");
+ } catch (TableNotFoundException e) {
+ // pass
+ }
+
+ bw1 = mtbw.getBatchWriter(newTable1);
+ bw2 = mtbw.getBatchWriter(newTable2);
+
+ Mutation m2 = new Mutation("bar");
+ m2.put("col1", "", "val1");
+ m2.put("col2", "", "val2");
+
+ bw1.addMutation(m2);
+ bw2.addMutation(m2);
+
+ mtbw.close();
+
+ Map<Entry<String,String>,String> expectations = new HashMap<Entry<String,String>,String>();
+ expectations.put(Maps.immutableEntry("foo", "col1"), "val1");
+ expectations.put(Maps.immutableEntry("foo", "col2"), "val2");
+ expectations.put(Maps.immutableEntry("bar", "col1"), "val1");
+ expectations.put(Maps.immutableEntry("bar", "col2"), "val2");
+
+ for (String table : Arrays.asList(newTable1, newTable2)) {
+ Scanner s = connector.createScanner(table, new Authorizations());
+ s.setRange(new Range());
+ Map<Entry<String,String>,String> actual = new HashMap<Entry<String,String>,String>();
+ for (Entry<Key,Value> entry : s) {
+ actual.put(Maps.immutableEntry(entry.getKey().getRow().toString(), entry.getKey().getColumnFamily().toString()), entry.getValue().toString());
+ }
+
+ Assert.assertEquals("Differing results for " + table, expectations, actual);
+ }
+ } finally {
+ if (null != mtbw) {
+ mtbw.close();
+ }
+ }
+ }
+
+ @Test
+ public void testTableRenameNewWritersNoCaching() throws Exception {
+ mtbw = getMultiTableBatchWriter(0);
+
+ try {
+ final String table1 = "testTableRenameNewWritersNoCaching_table1", table2 = "testTableRenameNewWritersNoCaching_table2";
+ final String newTable1 = "testTableRenameNewWritersNoCaching_newTable1", newTable2 = "testTableRenameNewWritersNoCaching_newTable2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ tops.rename(table1, newTable1);
+ tops.rename(table2, newTable2);
+
+ try {
+ bw1 = mtbw.getBatchWriter(table1);
+ Assert.fail("Should not have gotten batchwriter for " + table1);
+ } catch (TableNotFoundException e) {
+ // Pass
+ }
+
+ try {
+ bw2 = mtbw.getBatchWriter(table2);
+ } catch (TableNotFoundException e) {
+ // Pass
+ }
+ } finally {
+ if (null != mtbw) {
+ mtbw.close();
+ }
+ }
+ }
+
+ @Test
+ public void testTableDelete() throws Exception {
+ boolean mutationsRejected = false;
+
+ try {
+ final String table1 = "testTableDelete_table1", table2 = "testTableDelete_table2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ tops.delete(table1);
+ tops.delete(table2);
+
+ Mutation m2 = new Mutation("bar");
+ m2.put("col1", "", "val1");
+ m2.put("col2", "", "val2");
+
+ try {
+ bw1.addMutation(m2);
+ bw2.addMutation(m2);
+ } catch (MutationsRejectedException e) {
+ // Pass - Mutations might flush immediately
+ mutationsRejected = true;
+ }
+
+ } finally {
+ if (null != mtbw) {
+ try {
+ // Mutations might have flushed before the table offline occurred
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ // Pass
+ mutationsRejected = true;
+ }
+ }
+ }
+
+ Assert.assertTrue("Expected mutations to be rejected.", mutationsRejected);
+ }
+
+ @Test
+ public void testOfflineTable() throws Exception {
+ boolean mutationsRejected = false;
+
+ try {
+ final String table1 = "testOfflineTable_table1", table2 = "testOfflineTable_table2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ tops.offline(table1, true);
+ tops.offline(table2, true);
+
+ Mutation m2 = new Mutation("bar");
+ m2.put("col1", "", "val1");
+ m2.put("col2", "", "val2");
+
+ try {
+ bw1.addMutation(m2);
+ bw2.addMutation(m2);
+ } catch (MutationsRejectedException e) {
+ // Pass -- Mutations might flush immediately and fail because of offline table
+ mutationsRejected = true;
+ }
+ } finally {
+ if (null != mtbw) {
+ try {
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ // Pass
+ mutationsRejected = true;
+ }
+ }
+ }
+
+ Assert.assertTrue("Expected mutations to be rejected.", mutationsRejected);
+ }
+
+ @Test
+ public void testOfflineTableWithCache() throws Exception {
+ boolean mutationsRejected = false;
+
+ try {
+ final String table1 = "testOfflineTableWithCache_table1", table2 = "testOfflineTableWithCache_table2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ tops.offline(table1);
+
+ try {
+ bw1 = mtbw.getBatchWriter(table1);
+ } catch (TableOfflineException e) {
+ // pass
+ mutationsRejected = true;
+ }
+
+ tops.offline(table2);
+
+ try {
+ bw2 = mtbw.getBatchWriter(table2);
+ } catch (TableOfflineException e) {
+ // pass
+ mutationsRejected = true;
+ }
+ } finally {
+ if (null != mtbw) {
+ try {
+ // Mutations might have flushed before the table offline occurred
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ // Pass
+ mutationsRejected = true;
+ }
+ }
+ }
+
+ Assert.assertTrue("Expected mutations to be rejected.", mutationsRejected);
+ }
+
+ @Test
+ public void testOfflineTableWithoutCache() throws Exception {
+ mtbw = getMultiTableBatchWriter(0);
+ boolean mutationsRejected = false;
+
+ try {
+ final String table1 = "testOfflineTableWithoutCache_table1", table2 = "testOfflineTableWithoutCache_table2";
+
+ TableOperations tops = connector.tableOperations();
+ tops.create(table1);
+ tops.create(table2);
+
+ BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
+
+ Mutation m1 = new Mutation("foo");
+ m1.put("col1", "", "val1");
+ m1.put("col2", "", "val2");
+
+ bw1.addMutation(m1);
+ bw2.addMutation(m1);
+
+ // Mutations might or might not flush before tables goes offline
+ tops.offline(table1);
+ tops.offline(table2);
+
+ try {
+ bw1 = mtbw.getBatchWriter(table1);
+ Assert.fail(table1 + " should be offline");
+ } catch (TableOfflineException e) {
+ // pass
+ mutationsRejected = true;
+ }
+
+ try {
+ bw2 = mtbw.getBatchWriter(table2);
+ Assert.fail(table1 + " should be offline");
+ } catch (TableOfflineException e) {
+ // pass
+ mutationsRejected = true;
+ }
+ } finally {
+ if (null != mtbw) {
+ try {
+ // Mutations might have flushed before the table offline occurred
+ mtbw.close();
+ } catch (MutationsRejectedException e) {
+ // Pass
+ mutationsRejected = true;
+ }
+ }
+ }
+
+ Assert.assertTrue("Expected mutations to be rejected.", mutationsRejected);
+ }
+}